From 9b0e3a30920178e9c901cf49920a96a39d23b291 Mon Sep 17 00:00:00 2001 From: cmx Date: Wed, 25 Mar 2026 23:10:32 +0800 Subject: [PATCH] first commit --- LICENSE | 23 + Makefile | 73 + NOTICE | 58 + benchmark/BENCHMARK_GUIDELINES.md | 101 + benchmark/README.md | 48 + benchmark/__init__.py | 0 benchmark/benchmarks_visualizer.py | 299 + benchmark/data/all_benchmark_data.csv | 1957 + benchmark/scripts/__init__.py | 0 benchmark/scripts/benchmark_cpo_loss.py | 167 + benchmark/scripts/benchmark_cross_entropy.py | 126 + .../scripts/benchmark_distill_cosine_loss.py | 266 + .../scripts/benchmark_distill_jsd_loss.py | 272 + benchmark/scripts/benchmark_dpo_loss.py | 179 + benchmark/scripts/benchmark_dyt.py | 96 + benchmark/scripts/benchmark_embedding.py | 134 + .../scripts/benchmark_fused_add_rms_norm.py | 201 + .../benchmark_fused_linear_cross_entropy.py | 184 + .../scripts/benchmark_fused_linear_jsd.py | 260 + .../benchmark_fused_neighborhood_attention.py | 367 + benchmark/scripts/benchmark_geglu.py | 115 + benchmark/scripts/benchmark_group_norm.py | 137 + benchmark/scripts/benchmark_grpo_loss.py | 234 + benchmark/scripts/benchmark_jsd.py | 157 + benchmark/scripts/benchmark_kl_div.py | 117 + benchmark/scripts/benchmark_kto_loss.py | 314 + benchmark/scripts/benchmark_layer_norm.py | 125 + benchmark/scripts/benchmark_llama4_rope.py | 245 + benchmark/scripts/benchmark_mhc.py | 255 + benchmark/scripts/benchmark_mhc_lm.py | 455 + benchmark/scripts/benchmark_model_configs.py | 258 + .../benchmark_multi_token_attention.py | 218 + benchmark/scripts/benchmark_orpo_loss.py | 169 + benchmark/scripts/benchmark_poly_norm.py | 197 + benchmark/scripts/benchmark_qwen2vl_mrope.py | 241 + benchmark/scripts/benchmark_rms_norm.py | 162 + benchmark/scripts/benchmark_rope.py | 223 + benchmark/scripts/benchmark_simpo_loss.py | 167 + benchmark/scripts/benchmark_softmax.py | 140 + .../benchmark_sparse_multi_token_attention.py | 254 + benchmark/scripts/benchmark_sparsemax.py | 172 + benchmark/scripts/benchmark_swiglu.py | 115 + benchmark/scripts/benchmark_tiled_mlp.py | 397 + benchmark/scripts/benchmark_tvd.py | 145 + benchmark/scripts/utils.py | 439 + dev/fmt-requirements.txt | 1 + dev/modal/benchmarks.py | 73 + dev/modal/tests.py | 86 + docs/Examples.md | 268 + docs/Getting-Started.md | 64 + docs/High-Level-APIs.md | 93 + docs/Low-Level-APIs.md | 133 + docs/acknowledgement.md | 23 + docs/contributing.md | 114 + docs/images/banner.GIF | Bin 0 -> 716878 bytes docs/images/compose.gif | Bin 0 -> 522267 bytes docs/images/e2e-memory.png | Bin 0 -> 17055 bytes docs/images/e2e-tps.png | Bin 0 -> 16070 bytes docs/images/logo-banner.png | Bin 0 -> 139572 bytes docs/images/patch.gif | Bin 0 -> 415000 bytes docs/images/post-training.png | Bin 0 -> 148123 bytes docs/index.md | 186 + docs/license.md | 8 + examples/alignment/accelerate_config.yaml | 26 + examples/alignment/run_orpo.py | 35 + examples/huggingface/README.md | 55 + examples/huggingface/callback.py | 257 + examples/huggingface/config/fsdp_config.json | 5 + examples/huggingface/img/gemma_7b_mem.png | Bin 0 -> 12421 bytes examples/huggingface/img/gemma_7b_tp.png | Bin 0 -> 12127 bytes examples/huggingface/img/llama_mem_alloc.png | Bin 0 -> 15422 bytes examples/huggingface/img/llama_tps.png | Bin 0 -> 16117 bytes examples/huggingface/img/qwen_mem_alloc.png | Bin 0 -> 14025 bytes examples/huggingface/img/qwen_tps.png | Bin 0 -> 15408 bytes examples/huggingface/launch_on_modal.py | 69 + examples/huggingface/requirements.txt | 6 + examples/huggingface/run_benchmarks.sh | 52 + examples/huggingface/run_gemma.sh | 22 + examples/huggingface/run_llama.sh | 21 + examples/huggingface/run_qwen.sh | 22 + examples/huggingface/run_qwen2_vl.sh | 22 + examples/huggingface/training.py | 79 + examples/huggingface/training_multimodal.py | 169 + examples/lightning/README.md | 21 + examples/lightning/requirements.txt | 8 + examples/lightning/training.py | 281 + examples/medusa/README.md | 72 + examples/medusa/callback.py | 386 + .../docs/images/Memory_Stage1_num_head_3.png | Bin 0 -> 13454 bytes .../docs/images/Memory_Stage1_num_head_5.png | Bin 0 -> 13939 bytes .../docs/images/Memory_Stage2_num_head_3.png | Bin 0 -> 13069 bytes .../docs/images/Memory_Stage2_num_head_5.png | Bin 0 -> 13336 bytes .../images/Throughput_Stage1_num_head_3.png | Bin 0 -> 13950 bytes .../images/Throughput_Stage1_num_head_5.png | Bin 0 -> 14364 bytes .../images/Throughput_Stage2_num_head_3.png | Bin 0 -> 14905 bytes .../images/Throughput_Stage2_num_head_5.png | Bin 0 -> 13879 bytes examples/medusa/fsdp/acc-fsdp.conf | 24 + examples/medusa/medusa_util.py | 267 + examples/medusa/requirements.txt | 3 + examples/medusa/scripts/llama3_8b_medusa.sh | 53 + examples/medusa/train.py | 381 + licenses/LICENSE-Apache-2.0 | 201 + licenses/LICENSE-MIT-AutoAWQ | 21 + licenses/LICENSE-MIT-Efficient-Cross-Entropy | 21 + licenses/LICENSE-MIT-llmc | 21 + licenses/LICENSE-MIT-triton | 23 + mkdocs.yml | 87 + pyproject.toml | 86 + setup.py | 132 + src/liger_kernel/__init__.py | 0 src/liger_kernel/chunked_loss/README.md | 25 + src/liger_kernel/chunked_loss/__init__.py | 8 + .../chunked_loss/cosine_similarity_loss.py | 142 + src/liger_kernel/chunked_loss/cpo_loss.py | 157 + src/liger_kernel/chunked_loss/dpo_loss.py | 229 + src/liger_kernel/chunked_loss/functional.py | 17 + .../chunked_loss/fused_linear_distillation.py | 299 + .../chunked_loss/fused_linear_ppo.py | 421 + .../chunked_loss/fused_linear_preference.py | 433 + .../fused_linear_unpaired_preference.py | 341 + src/liger_kernel/chunked_loss/grpo_loss.py | 462 + src/liger_kernel/chunked_loss/jsd_loss.py | 215 + src/liger_kernel/chunked_loss/kto_loss.py | 210 + src/liger_kernel/chunked_loss/orpo_loss.py | 144 + src/liger_kernel/chunked_loss/simpo_loss.py | 165 + src/liger_kernel/env_report.py | 63 + src/liger_kernel/ops/__init__.py | 144 + src/liger_kernel/ops/backends/README.md | 151 + src/liger_kernel/ops/backends/__init__.py | 13 + .../ops/backends/_ascend/__init__.py | 5 + .../_ascend/ascend-ub-manager-design.md | 492 + .../ops/backends/_ascend/ops/__init__.py | 139 + .../ops/backends/_ascend/ops/cross_entropy.py | 568 + .../ops/backends/_ascend/ops/dyt.py | 285 + .../ops/backends/_ascend/ops/embedding.py | 210 + .../_ascend/ops/fused_add_rms_norm.py | 781 + .../backends/_ascend/ops/fused_linear_jsd.py | 227 + .../ops/backends/_ascend/ops/geglu.py | 187 + .../ops/backends/_ascend/ops/group_norm.py | 474 + .../ops/backends/_ascend/ops/grpo_loss.py | 1006 + .../ops/backends/_ascend/ops/jsd.py | 229 + .../ops/backends/_ascend/ops/kl_div.py | 327 + .../ops/backends/_ascend/ops/layer_norm.py | 642 + .../ops/backends/_ascend/ops/llama4_rope.py | 306 + .../ops/backends/_ascend/ops/poly_norm.py | 786 + .../ops/backends/_ascend/ops/qwen2vl_mrope.py | 272 + .../ops/backends/_ascend/ops/rms_norm.py | 782 + .../ops/backends/_ascend/ops/rope.py | 262 + .../ops/backends/_ascend/ops/softmax.py | 344 + .../ops/backends/_ascend/ops/sparsemax.py | 385 + .../ops/backends/_ascend/ops/swiglu.py | 136 + .../ops/backends/_ascend/ops/tvd.py | 221 + .../ops/backends/_ascend/ub_manager.py | 373 + src/liger_kernel/ops/backends/registry.py | 61 + src/liger_kernel/ops/cross_entropy.py | 558 + src/liger_kernel/ops/dyt.py | 160 + .../ops/experimental/embedding.py | 141 + .../ops/experimental/mm_int8int2.py | 349 + src/liger_kernel/ops/fused_add_rms_norm.py | 410 + .../ops/fused_linear_cross_entropy.py | 400 + src/liger_kernel/ops/fused_linear_jsd.py | 228 + .../ops/fused_neighborhood_attention.py | 1022 + src/liger_kernel/ops/geglu.py | 143 + src/liger_kernel/ops/group_norm.py | 311 + src/liger_kernel/ops/grpo_loss.py | 930 + src/liger_kernel/ops/jsd.py | 201 + src/liger_kernel/ops/kl_div.py | 259 + src/liger_kernel/ops/layer_norm.py | 320 + src/liger_kernel/ops/llama4_rope.py | 180 + src/liger_kernel/ops/mhc.py | 1674 + src/liger_kernel/ops/multi_token_attention.py | 207 + src/liger_kernel/ops/poly_norm.py | 384 + src/liger_kernel/ops/qwen2vl_mrope.py | 222 + src/liger_kernel/ops/rms_norm.py | 654 + src/liger_kernel/ops/rope.py | 239 + src/liger_kernel/ops/softmax.py | 201 + src/liger_kernel/ops/sparsemax.py | 177 + src/liger_kernel/ops/swiglu.py | 151 + src/liger_kernel/ops/tiled_mlp.py | 136 + src/liger_kernel/ops/tvd.py | 218 + src/liger_kernel/ops/utils.py | 152 + src/liger_kernel/transformers/__init__.py | 233 + src/liger_kernel/transformers/auto_model.py | 59 + .../transformers/cross_entropy.py | 61 + src/liger_kernel/transformers/dyt.py | 22 + .../transformers/experimental/__init__.py | 5 + .../transformers/experimental/embedding.py | 26 + src/liger_kernel/transformers/fsdp.py | 55 + src/liger_kernel/transformers/functional.py | 410 + .../transformers/fused_add_rms_norm.py | 39 + .../fused_linear_cross_entropy.py | 69 + .../transformers/fused_linear_jsd.py | 95 + .../fused_neighborhood_attention.py | 234 + src/liger_kernel/transformers/geglu.py | 22 + src/liger_kernel/transformers/group_norm.py | 50 + src/liger_kernel/transformers/grpo_loss.py | 206 + src/liger_kernel/transformers/jsd.py | 70 + src/liger_kernel/transformers/kl_div.py | 12 + src/liger_kernel/transformers/layer_norm.py | 24 + src/liger_kernel/transformers/llama4_rope.py | 93 + src/liger_kernel/transformers/mhc.py | 162 + .../transformers/model/__init__.py | 0 .../transformers/model/exaone4.py | 139 + .../transformers/model/falcon_h1.py | 125 + src/liger_kernel/transformers/model/gemma.py | 144 + src/liger_kernel/transformers/model/gemma2.py | 157 + src/liger_kernel/transformers/model/gemma3.py | 343 + src/liger_kernel/transformers/model/glm4.py | 141 + src/liger_kernel/transformers/model/glm4v.py | 165 + .../transformers/model/glm4v_moe.py | 174 + .../transformers/model/gpt_oss.py | 213 + .../transformers/model/hunyuan_v1.py | 137 + .../transformers/model/internvl.py | 160 + src/liger_kernel/transformers/model/llama.py | 202 + src/liger_kernel/transformers/model/llama4.py | 124 + src/liger_kernel/transformers/model/llava.py | 160 + .../transformers/model/loss_utils.py | 106 + .../transformers/model/mistral.py | 146 + .../transformers/model/mixtral.py | 167 + src/liger_kernel/transformers/model/mllama.py | 149 + src/liger_kernel/transformers/model/olmo2.py | 141 + src/liger_kernel/transformers/model/olmo3.py | 143 + .../transformers/model/output_classes.py | 173 + .../transformers/model/paligemma.py | 250 + src/liger_kernel/transformers/model/phi3.py | 123 + .../transformers/model/pixtral.py | 4 + src/liger_kernel/transformers/model/qwen2.py | 260 + .../transformers/model/qwen2_5_vl.py | 186 + .../transformers/model/qwen2_vl.py | 182 + src/liger_kernel/transformers/model/qwen3.py | 139 + .../transformers/model/qwen3_5.py | 256 + .../transformers/model/qwen3_5_moe.py | 157 + .../transformers/model/qwen3_moe.py | 155 + .../transformers/model/qwen3_next.py | 155 + .../transformers/model/qwen3_vl.py | 155 + .../transformers/model/qwen3_vl_moe.py | 131 + .../transformers/model/smollm3.py | 200 + .../transformers/model/smolvlm.py | 158 + src/liger_kernel/transformers/monkey_patch.py | 3178 ++ .../transformers/multi_token_attention.py | 64 + src/liger_kernel/transformers/poly_norm.py | 42 + .../transformers/qwen2vl_mrope.py | 20 + src/liger_kernel/transformers/rms_norm.py | 91 + src/liger_kernel/transformers/rope.py | 64 + src/liger_kernel/transformers/softmax.py | 12 + src/liger_kernel/transformers/sparsemax.py | 16 + src/liger_kernel/transformers/swiglu.py | 145 + src/liger_kernel/transformers/tiled_mlp.py | 125 + .../transformers/trainer/__init__.py | 4 + .../transformers/trainer/orpo_trainer.py | 130 + .../transformers/trainer_integration.py | 2 + src/liger_kernel/transformers/tvd.py | 13 + src/liger_kernel/triton/__init__.py | 1 + src/liger_kernel/triton/monkey_patch.py | 40 + src/liger_kernel/utils.py | 125 + test/__init__.py | 0 test/chunked_loss/__init__.py | 0 test/chunked_loss/test_cosine_loss.py | 320 + test/chunked_loss/test_cpo_loss.py | 302 + test/chunked_loss/test_dpo_loss.py | 938 + test/chunked_loss/test_grpo_loss.py | 993 + test/chunked_loss/test_jsd_loss.py | 441 + test/chunked_loss/test_kto_loss.py | 434 + test/chunked_loss/test_orpo_loss.py | 266 + test/chunked_loss/test_simpo_loss.py | 215 + test/conftest.py | 15 + test/convergence/__init__.py | 0 test/convergence/bf16/__init__.py | 0 test/convergence/bf16/test_mini_models.py | 2324 + .../bf16/test_mini_models_multimodal.py | 1830 + .../bf16/test_mini_models_with_logits.py | 2167 + test/convergence/fp32/__init__.py | 0 test/convergence/fp32/test_mini_models.py | 2170 + .../fp32/test_mini_models_multimodal.py | 1934 + .../fp32/test_mini_models_with_logits.py | 2038 + .../gemma-3-4b-it/tokenizer_config.json | 90 + .../paligemma-3b-pt-224/tokenizer_config.json | 61 + .../tokenizer_config.json | 1192 + .../llava-1.5-7b-hf/preprocessor_config.json | 28 + .../llava-1.5-7b-hf/processor_config.json | 7 + .../llava-1.5-7b-hf/tokenizer_config.json | 66 + .../InternVL3-1B-hf/tokenizer_config.json | 307 + .../tokenizer_config.json | 63 + .../tokenizer_config.json | 63 + .../tokenizer_config.json | 63 + .../tokenizer_config.json | 31 + .../tokenizer_config.json | 98 + .../scripts/generate_tokenized_dataset.py | 71 + test/resources/tiny_shakespeare.txt | 40000 ++++++++++++++++ .../data-00000-of-00001.arrow | Bin 0 -> 25932712 bytes .../dataset_info.json | 48 + .../tiny_shakespeare_tokenized/state.json | 13 + test/transformers/test_auto_model.py | 89 + test/transformers/test_cross_entropy.py | 1267 + test/transformers/test_dyt.py | 160 + test/transformers/test_embedding.py | 61 + test/transformers/test_flex_attention.py | 291 + test/transformers/test_fused_add_rms_norm.py | 219 + .../test_fused_linear_cross_entropy.py | 1024 + test/transformers/test_fused_linear_jsd.py | 450 + .../test_fused_neighborhood_attention.py | 572 + test/transformers/test_geglu.py | 264 + test/transformers/test_group_norm.py | 52 + test/transformers/test_grpo_loss.py | 1331 + test/transformers/test_jsd.py | 352 + test/transformers/test_kl_div.py | 104 + test/transformers/test_layer_norm.py | 116 + test/transformers/test_llama4_rope.py | 149 + test/transformers/test_mhc.py | 522 + test/transformers/test_mm_int8int2.py | 97 + test/transformers/test_monkey_patch.py | 3209 ++ .../test_multi_token_attention.py | 327 + test/transformers/test_poly_norm.py | 281 + test/transformers/test_qwen2vl_mrope.py | 139 + test/transformers/test_rms_norm.py | 311 + test/transformers/test_rope.py | 183 + test/transformers/test_softmax.py | 103 + test/transformers/test_sparsemax.py | 111 + test/transformers/test_swiglu.py | 492 + test/transformers/test_tiled_mlp.py | 197 + test/transformers/test_trainer_integration.py | 8 + test/transformers/test_transformers.py | 17 + test/transformers/test_tvd.py | 188 + test/triton/test_triton_monkey_patch.py | 29 + test/utils.py | 1158 + 325 files changed, 121805 insertions(+) create mode 100755 LICENSE create mode 100755 Makefile create mode 100755 NOTICE create mode 100755 benchmark/BENCHMARK_GUIDELINES.md create mode 100755 benchmark/README.md create mode 100755 benchmark/__init__.py create mode 100755 benchmark/benchmarks_visualizer.py create mode 100755 benchmark/data/all_benchmark_data.csv create mode 100755 benchmark/scripts/__init__.py create mode 100755 benchmark/scripts/benchmark_cpo_loss.py create mode 100755 benchmark/scripts/benchmark_cross_entropy.py create mode 100755 benchmark/scripts/benchmark_distill_cosine_loss.py create mode 100755 benchmark/scripts/benchmark_distill_jsd_loss.py create mode 100755 benchmark/scripts/benchmark_dpo_loss.py create mode 100755 benchmark/scripts/benchmark_dyt.py create mode 100755 benchmark/scripts/benchmark_embedding.py create mode 100755 benchmark/scripts/benchmark_fused_add_rms_norm.py create mode 100755 benchmark/scripts/benchmark_fused_linear_cross_entropy.py create mode 100755 benchmark/scripts/benchmark_fused_linear_jsd.py create mode 100755 benchmark/scripts/benchmark_fused_neighborhood_attention.py create mode 100755 benchmark/scripts/benchmark_geglu.py create mode 100755 benchmark/scripts/benchmark_group_norm.py create mode 100755 benchmark/scripts/benchmark_grpo_loss.py create mode 100755 benchmark/scripts/benchmark_jsd.py create mode 100755 benchmark/scripts/benchmark_kl_div.py create mode 100755 benchmark/scripts/benchmark_kto_loss.py create mode 100755 benchmark/scripts/benchmark_layer_norm.py create mode 100755 benchmark/scripts/benchmark_llama4_rope.py create mode 100755 benchmark/scripts/benchmark_mhc.py create mode 100755 benchmark/scripts/benchmark_mhc_lm.py create mode 100755 benchmark/scripts/benchmark_model_configs.py create mode 100755 benchmark/scripts/benchmark_multi_token_attention.py create mode 100755 benchmark/scripts/benchmark_orpo_loss.py create mode 100755 benchmark/scripts/benchmark_poly_norm.py create mode 100755 benchmark/scripts/benchmark_qwen2vl_mrope.py create mode 100755 benchmark/scripts/benchmark_rms_norm.py create mode 100755 benchmark/scripts/benchmark_rope.py create mode 100755 benchmark/scripts/benchmark_simpo_loss.py create mode 100755 benchmark/scripts/benchmark_softmax.py create mode 100755 benchmark/scripts/benchmark_sparse_multi_token_attention.py create mode 100755 benchmark/scripts/benchmark_sparsemax.py create mode 100755 benchmark/scripts/benchmark_swiglu.py create mode 100755 benchmark/scripts/benchmark_tiled_mlp.py create mode 100755 benchmark/scripts/benchmark_tvd.py create mode 100755 benchmark/scripts/utils.py create mode 100755 dev/fmt-requirements.txt create mode 100755 dev/modal/benchmarks.py create mode 100755 dev/modal/tests.py create mode 100755 docs/Examples.md create mode 100755 docs/Getting-Started.md create mode 100755 docs/High-Level-APIs.md create mode 100755 docs/Low-Level-APIs.md create mode 100755 docs/acknowledgement.md create mode 100755 docs/contributing.md create mode 100755 docs/images/banner.GIF create mode 100755 docs/images/compose.gif create mode 100755 docs/images/e2e-memory.png create mode 100755 docs/images/e2e-tps.png create mode 100755 docs/images/logo-banner.png create mode 100755 docs/images/patch.gif create mode 100755 docs/images/post-training.png create mode 100755 docs/index.md create mode 100755 docs/license.md create mode 100755 examples/alignment/accelerate_config.yaml create mode 100755 examples/alignment/run_orpo.py create mode 100755 examples/huggingface/README.md create mode 100755 examples/huggingface/callback.py create mode 100755 examples/huggingface/config/fsdp_config.json create mode 100755 examples/huggingface/img/gemma_7b_mem.png create mode 100755 examples/huggingface/img/gemma_7b_tp.png create mode 100755 examples/huggingface/img/llama_mem_alloc.png create mode 100755 examples/huggingface/img/llama_tps.png create mode 100755 examples/huggingface/img/qwen_mem_alloc.png create mode 100755 examples/huggingface/img/qwen_tps.png create mode 100755 examples/huggingface/launch_on_modal.py create mode 100755 examples/huggingface/requirements.txt create mode 100755 examples/huggingface/run_benchmarks.sh create mode 100755 examples/huggingface/run_gemma.sh create mode 100755 examples/huggingface/run_llama.sh create mode 100755 examples/huggingface/run_qwen.sh create mode 100755 examples/huggingface/run_qwen2_vl.sh create mode 100755 examples/huggingface/training.py create mode 100755 examples/huggingface/training_multimodal.py create mode 100755 examples/lightning/README.md create mode 100755 examples/lightning/requirements.txt create mode 100755 examples/lightning/training.py create mode 100755 examples/medusa/README.md create mode 100755 examples/medusa/callback.py create mode 100755 examples/medusa/docs/images/Memory_Stage1_num_head_3.png create mode 100755 examples/medusa/docs/images/Memory_Stage1_num_head_5.png create mode 100755 examples/medusa/docs/images/Memory_Stage2_num_head_3.png create mode 100755 examples/medusa/docs/images/Memory_Stage2_num_head_5.png create mode 100755 examples/medusa/docs/images/Throughput_Stage1_num_head_3.png create mode 100755 examples/medusa/docs/images/Throughput_Stage1_num_head_5.png create mode 100755 examples/medusa/docs/images/Throughput_Stage2_num_head_3.png create mode 100755 examples/medusa/docs/images/Throughput_Stage2_num_head_5.png create mode 100755 examples/medusa/fsdp/acc-fsdp.conf create mode 100755 examples/medusa/medusa_util.py create mode 100755 examples/medusa/requirements.txt create mode 100755 examples/medusa/scripts/llama3_8b_medusa.sh create mode 100755 examples/medusa/train.py create mode 100755 licenses/LICENSE-Apache-2.0 create mode 100755 licenses/LICENSE-MIT-AutoAWQ create mode 100755 licenses/LICENSE-MIT-Efficient-Cross-Entropy create mode 100755 licenses/LICENSE-MIT-llmc create mode 100755 licenses/LICENSE-MIT-triton create mode 100755 mkdocs.yml create mode 100755 pyproject.toml create mode 100755 setup.py create mode 100755 src/liger_kernel/__init__.py create mode 100755 src/liger_kernel/chunked_loss/README.md create mode 100755 src/liger_kernel/chunked_loss/__init__.py create mode 100755 src/liger_kernel/chunked_loss/cosine_similarity_loss.py create mode 100755 src/liger_kernel/chunked_loss/cpo_loss.py create mode 100755 src/liger_kernel/chunked_loss/dpo_loss.py create mode 100755 src/liger_kernel/chunked_loss/functional.py create mode 100755 src/liger_kernel/chunked_loss/fused_linear_distillation.py create mode 100755 src/liger_kernel/chunked_loss/fused_linear_ppo.py create mode 100755 src/liger_kernel/chunked_loss/fused_linear_preference.py create mode 100755 src/liger_kernel/chunked_loss/fused_linear_unpaired_preference.py create mode 100755 src/liger_kernel/chunked_loss/grpo_loss.py create mode 100755 src/liger_kernel/chunked_loss/jsd_loss.py create mode 100755 src/liger_kernel/chunked_loss/kto_loss.py create mode 100755 src/liger_kernel/chunked_loss/orpo_loss.py create mode 100755 src/liger_kernel/chunked_loss/simpo_loss.py create mode 100755 src/liger_kernel/env_report.py create mode 100755 src/liger_kernel/ops/__init__.py create mode 100755 src/liger_kernel/ops/backends/README.md create mode 100755 src/liger_kernel/ops/backends/__init__.py create mode 100755 src/liger_kernel/ops/backends/_ascend/__init__.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ascend-ub-manager-design.md create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/__init__.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/cross_entropy.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/dyt.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/embedding.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/fused_add_rms_norm.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/fused_linear_jsd.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/geglu.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/group_norm.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/grpo_loss.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/jsd.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/kl_div.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/layer_norm.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/llama4_rope.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/poly_norm.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/qwen2vl_mrope.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/rms_norm.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/rope.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/softmax.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/sparsemax.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/swiglu.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ops/tvd.py create mode 100755 src/liger_kernel/ops/backends/_ascend/ub_manager.py create mode 100755 src/liger_kernel/ops/backends/registry.py create mode 100755 src/liger_kernel/ops/cross_entropy.py create mode 100755 src/liger_kernel/ops/dyt.py create mode 100755 src/liger_kernel/ops/experimental/embedding.py create mode 100755 src/liger_kernel/ops/experimental/mm_int8int2.py create mode 100755 src/liger_kernel/ops/fused_add_rms_norm.py create mode 100755 src/liger_kernel/ops/fused_linear_cross_entropy.py create mode 100755 src/liger_kernel/ops/fused_linear_jsd.py create mode 100755 src/liger_kernel/ops/fused_neighborhood_attention.py create mode 100755 src/liger_kernel/ops/geglu.py create mode 100755 src/liger_kernel/ops/group_norm.py create mode 100755 src/liger_kernel/ops/grpo_loss.py create mode 100755 src/liger_kernel/ops/jsd.py create mode 100755 src/liger_kernel/ops/kl_div.py create mode 100755 src/liger_kernel/ops/layer_norm.py create mode 100755 src/liger_kernel/ops/llama4_rope.py create mode 100755 src/liger_kernel/ops/mhc.py create mode 100755 src/liger_kernel/ops/multi_token_attention.py create mode 100755 src/liger_kernel/ops/poly_norm.py create mode 100755 src/liger_kernel/ops/qwen2vl_mrope.py create mode 100755 src/liger_kernel/ops/rms_norm.py create mode 100755 src/liger_kernel/ops/rope.py create mode 100755 src/liger_kernel/ops/softmax.py create mode 100755 src/liger_kernel/ops/sparsemax.py create mode 100755 src/liger_kernel/ops/swiglu.py create mode 100755 src/liger_kernel/ops/tiled_mlp.py create mode 100755 src/liger_kernel/ops/tvd.py create mode 100755 src/liger_kernel/ops/utils.py create mode 100755 src/liger_kernel/transformers/__init__.py create mode 100755 src/liger_kernel/transformers/auto_model.py create mode 100755 src/liger_kernel/transformers/cross_entropy.py create mode 100755 src/liger_kernel/transformers/dyt.py create mode 100755 src/liger_kernel/transformers/experimental/__init__.py create mode 100755 src/liger_kernel/transformers/experimental/embedding.py create mode 100755 src/liger_kernel/transformers/fsdp.py create mode 100755 src/liger_kernel/transformers/functional.py create mode 100755 src/liger_kernel/transformers/fused_add_rms_norm.py create mode 100755 src/liger_kernel/transformers/fused_linear_cross_entropy.py create mode 100755 src/liger_kernel/transformers/fused_linear_jsd.py create mode 100755 src/liger_kernel/transformers/fused_neighborhood_attention.py create mode 100755 src/liger_kernel/transformers/geglu.py create mode 100755 src/liger_kernel/transformers/group_norm.py create mode 100755 src/liger_kernel/transformers/grpo_loss.py create mode 100755 src/liger_kernel/transformers/jsd.py create mode 100755 src/liger_kernel/transformers/kl_div.py create mode 100755 src/liger_kernel/transformers/layer_norm.py create mode 100755 src/liger_kernel/transformers/llama4_rope.py create mode 100755 src/liger_kernel/transformers/mhc.py create mode 100755 src/liger_kernel/transformers/model/__init__.py create mode 100755 src/liger_kernel/transformers/model/exaone4.py create mode 100755 src/liger_kernel/transformers/model/falcon_h1.py create mode 100755 src/liger_kernel/transformers/model/gemma.py create mode 100755 src/liger_kernel/transformers/model/gemma2.py create mode 100755 src/liger_kernel/transformers/model/gemma3.py create mode 100755 src/liger_kernel/transformers/model/glm4.py create mode 100755 src/liger_kernel/transformers/model/glm4v.py create mode 100755 src/liger_kernel/transformers/model/glm4v_moe.py create mode 100755 src/liger_kernel/transformers/model/gpt_oss.py create mode 100755 src/liger_kernel/transformers/model/hunyuan_v1.py create mode 100755 src/liger_kernel/transformers/model/internvl.py create mode 100755 src/liger_kernel/transformers/model/llama.py create mode 100755 src/liger_kernel/transformers/model/llama4.py create mode 100755 src/liger_kernel/transformers/model/llava.py create mode 100755 src/liger_kernel/transformers/model/loss_utils.py create mode 100755 src/liger_kernel/transformers/model/mistral.py create mode 100755 src/liger_kernel/transformers/model/mixtral.py create mode 100755 src/liger_kernel/transformers/model/mllama.py create mode 100755 src/liger_kernel/transformers/model/olmo2.py create mode 100755 src/liger_kernel/transformers/model/olmo3.py create mode 100755 src/liger_kernel/transformers/model/output_classes.py create mode 100755 src/liger_kernel/transformers/model/paligemma.py create mode 100755 src/liger_kernel/transformers/model/phi3.py create mode 100755 src/liger_kernel/transformers/model/pixtral.py create mode 100755 src/liger_kernel/transformers/model/qwen2.py create mode 100755 src/liger_kernel/transformers/model/qwen2_5_vl.py create mode 100755 src/liger_kernel/transformers/model/qwen2_vl.py create mode 100755 src/liger_kernel/transformers/model/qwen3.py create mode 100755 src/liger_kernel/transformers/model/qwen3_5.py create mode 100755 src/liger_kernel/transformers/model/qwen3_5_moe.py create mode 100755 src/liger_kernel/transformers/model/qwen3_moe.py create mode 100755 src/liger_kernel/transformers/model/qwen3_next.py create mode 100755 src/liger_kernel/transformers/model/qwen3_vl.py create mode 100755 src/liger_kernel/transformers/model/qwen3_vl_moe.py create mode 100755 src/liger_kernel/transformers/model/smollm3.py create mode 100755 src/liger_kernel/transformers/model/smolvlm.py create mode 100755 src/liger_kernel/transformers/monkey_patch.py create mode 100755 src/liger_kernel/transformers/multi_token_attention.py create mode 100755 src/liger_kernel/transformers/poly_norm.py create mode 100755 src/liger_kernel/transformers/qwen2vl_mrope.py create mode 100755 src/liger_kernel/transformers/rms_norm.py create mode 100755 src/liger_kernel/transformers/rope.py create mode 100755 src/liger_kernel/transformers/softmax.py create mode 100755 src/liger_kernel/transformers/sparsemax.py create mode 100755 src/liger_kernel/transformers/swiglu.py create mode 100755 src/liger_kernel/transformers/tiled_mlp.py create mode 100755 src/liger_kernel/transformers/trainer/__init__.py create mode 100755 src/liger_kernel/transformers/trainer/orpo_trainer.py create mode 100755 src/liger_kernel/transformers/trainer_integration.py create mode 100755 src/liger_kernel/transformers/tvd.py create mode 100755 src/liger_kernel/triton/__init__.py create mode 100755 src/liger_kernel/triton/monkey_patch.py create mode 100755 src/liger_kernel/utils.py create mode 100755 test/__init__.py create mode 100755 test/chunked_loss/__init__.py create mode 100755 test/chunked_loss/test_cosine_loss.py create mode 100755 test/chunked_loss/test_cpo_loss.py create mode 100755 test/chunked_loss/test_dpo_loss.py create mode 100755 test/chunked_loss/test_grpo_loss.py create mode 100755 test/chunked_loss/test_jsd_loss.py create mode 100755 test/chunked_loss/test_kto_loss.py create mode 100755 test/chunked_loss/test_orpo_loss.py create mode 100755 test/chunked_loss/test_simpo_loss.py create mode 100755 test/conftest.py create mode 100755 test/convergence/__init__.py create mode 100755 test/convergence/bf16/__init__.py create mode 100755 test/convergence/bf16/test_mini_models.py create mode 100755 test/convergence/bf16/test_mini_models_multimodal.py create mode 100755 test/convergence/bf16/test_mini_models_with_logits.py create mode 100755 test/convergence/fp32/__init__.py create mode 100755 test/convergence/fp32/test_mini_models.py create mode 100755 test/convergence/fp32/test_mini_models_multimodal.py create mode 100755 test/convergence/fp32/test_mini_models_with_logits.py create mode 100755 test/resources/fake_configs/Google/Gemma3/gemma-3-4b-it/tokenizer_config.json create mode 100755 test/resources/fake_configs/Google/Paligemma/paligemma-3b-pt-224/tokenizer_config.json create mode 100755 test/resources/fake_configs/HuggingFaceTB/SmolVLM2-256M-Video-Instruct/tokenizer_config.json create mode 100755 test/resources/fake_configs/Llava/llava-1.5-7b-hf/preprocessor_config.json create mode 100755 test/resources/fake_configs/Llava/llava-1.5-7b-hf/processor_config.json create mode 100755 test/resources/fake_configs/Llava/llava-1.5-7b-hf/tokenizer_config.json create mode 100755 test/resources/fake_configs/OpenGVLab/InternVL3-1B-hf/tokenizer_config.json create mode 100755 test/resources/fake_configs/Qwen/Qwen2-VL-7B-Instruct/tokenizer_config.json create mode 100755 test/resources/fake_configs/Qwen/Qwen2.5-VL-7B-Instruct/tokenizer_config.json create mode 100755 test/resources/fake_configs/Qwen/Qwen3-VL-4B-Instruct/tokenizer_config.json create mode 100755 test/resources/fake_configs/meta-llama/Llama-3.2-11B-Vision-Instruct/tokenizer_config.json create mode 100755 test/resources/fake_configs/meta-llama/Llama-4-Scout-17B-16E-Instruct/tokenizer_config.json create mode 100755 test/resources/scripts/generate_tokenized_dataset.py create mode 100755 test/resources/tiny_shakespeare.txt create mode 100755 test/resources/tiny_shakespeare_tokenized/data-00000-of-00001.arrow create mode 100755 test/resources/tiny_shakespeare_tokenized/dataset_info.json create mode 100755 test/resources/tiny_shakespeare_tokenized/state.json create mode 100755 test/transformers/test_auto_model.py create mode 100755 test/transformers/test_cross_entropy.py create mode 100755 test/transformers/test_dyt.py create mode 100755 test/transformers/test_embedding.py create mode 100755 test/transformers/test_flex_attention.py create mode 100755 test/transformers/test_fused_add_rms_norm.py create mode 100755 test/transformers/test_fused_linear_cross_entropy.py create mode 100755 test/transformers/test_fused_linear_jsd.py create mode 100755 test/transformers/test_fused_neighborhood_attention.py create mode 100755 test/transformers/test_geglu.py create mode 100755 test/transformers/test_group_norm.py create mode 100755 test/transformers/test_grpo_loss.py create mode 100755 test/transformers/test_jsd.py create mode 100755 test/transformers/test_kl_div.py create mode 100755 test/transformers/test_layer_norm.py create mode 100755 test/transformers/test_llama4_rope.py create mode 100755 test/transformers/test_mhc.py create mode 100755 test/transformers/test_mm_int8int2.py create mode 100755 test/transformers/test_monkey_patch.py create mode 100755 test/transformers/test_multi_token_attention.py create mode 100755 test/transformers/test_poly_norm.py create mode 100755 test/transformers/test_qwen2vl_mrope.py create mode 100755 test/transformers/test_rms_norm.py create mode 100755 test/transformers/test_rope.py create mode 100755 test/transformers/test_softmax.py create mode 100755 test/transformers/test_sparsemax.py create mode 100755 test/transformers/test_swiglu.py create mode 100755 test/transformers/test_tiled_mlp.py create mode 100755 test/transformers/test_trainer_integration.py create mode 100755 test/transformers/test_transformers.py create mode 100755 test/transformers/test_tvd.py create mode 100755 test/triton/test_triton_monkey_patch.py create mode 100755 test/utils.py diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..d2fcc2b --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +BSD 2-CLAUSE LICENSE +Copyright 2024 LinkedIn Corporation +All Rights Reserved. +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided +with the distribution. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100755 index 0000000..904d3c1 --- /dev/null +++ b/Makefile @@ -0,0 +1,73 @@ +.PHONY: test checkstyle test-convergence all serve build clean + + +all: checkstyle test test-convergence + +# Command to run pytest for correctness tests +test: + python -m pytest --disable-warnings \ + --cov=src/liger_kernel \ + --cov-report=term-missing \ + --ignore=test/convergence \ + test/ + +# Command to run coverage report +coverage: + coverage report -m + +# Command to run ruff for linting and formatting code +checkstyle: + ruff check --output-format=concise .; ruff_check_status=$$?; \ + ruff format --check --diff .; ruff_format_status=$$?; \ + ruff check . --fix; \ + ruff format .; \ + if [ $$ruff_check_status -ne 0 ] || [ $$ruff_format_status -ne 0 ]; then \ + exit 1; \ + fi + +# Command to run pytest for convergence tests +# We have to explicitly set HF_DATASETS_OFFLINE=1, or dataset will silently try to send metrics and timeout (80s) https://github.com/huggingface/datasets/blob/37a603679f451826cfafd8aae00738b01dcb9d58/src/datasets/load.py#L286 +test-convergence: + HF_DATASETS_OFFLINE=1 python -m pytest --disable-warnings test/convergence/fp32/test_mini_models.py + HF_DATASETS_OFFLINE=1 python -m pytest --disable-warnings test/convergence/fp32/test_mini_models_multimodal.py + HF_DATASETS_OFFLINE=1 python -m pytest --disable-warnings test/convergence/fp32/test_mini_models_with_logits.py + + HF_DATASETS_OFFLINE=1 python -m pytest --disable-warnings test/convergence/bf16/test_mini_models.py + HF_DATASETS_OFFLINE=1 python -m pytest --disable-warnings test/convergence/bf16/test_mini_models_multimodal.py + HF_DATASETS_OFFLINE=1 python -m pytest --disable-warnings test/convergence/bf16/test_mini_models_with_logits.py + +# Command to run all benchmark scripts and update benchmarking data file +# By default this doesn't overwrite existing data for the same benchmark experiment +# run with `make run-benchmarks OVERWRITE=1` to overwrite existing benchmark data +BENCHMARK_DIR = benchmark/scripts +BENCHMARK_SCRIPTS = $(wildcard $(BENCHMARK_DIR)/benchmark_*.py) +OVERWRITE ?= 0 + +run-benchmarks: + @for script in $(BENCHMARK_SCRIPTS); do \ + echo "Running benchmark: $$script"; \ + if [ $(OVERWRITE) -eq 1 ]; then \ + python $$script --overwrite; \ + else \ + python $$script; \ + fi; \ + done + +# MkDocs Configuration +MKDOCS = mkdocs +CONFIG_FILE = mkdocs.yml +SITE_DIR = site + +# MkDocs targets + +# Serve the documentation +serve: + $(MKDOCS) serve -f $(CONFIG_FILE) + +# Build the documentation into the specified site directory +build: + $(MKDOCS) build -f $(CONFIG_FILE) --site-dir $(SITE_DIR) + +# Clean the output directory +clean: + rm -rf $(SITE_DIR)/ diff --git a/NOTICE b/NOTICE new file mode 100755 index 0000000..ea28817 --- /dev/null +++ b/NOTICE @@ -0,0 +1,58 @@ +Copyright 2024 LinkedIn Corporation +All Rights Reserved. + +Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + +This product includes software developed by LinkedIn Corporation. + +This product contains code derived from the following open source projects: + +1. Unsloth + Copyright (c) 2023 Unsloth AI + Licensed under the Apache License, Version 2.0 + Source: https://github.com/unslothai/unsloth + + The `calculate_settings` function to determine block size and warp is reused for Norm and MLP operations. + Modifications and additions were made to the RMS Norm implementation. + +2. Triton + Copyright (c) 2023 OpenAI + Licensed under the MIT License + Source: https://github.com/openai/triton + + Modifications were made based on Triton tutorials for the RMS Norm implementation. + +3. Efficient Cross Entropy + Copyright (c) 2023 Mohamed Malek + Licensed under the MIT License + Source: https://github.com/mgmalek/efficient_cross_entropy + + The idea of gradient-in-forward and chunking was used in the Linear Cross Entropy implementation. + +4. Flash Attention + Copyright (c) 2023 Tri Dao, Daniel Y. Fu, Stefano Ermon, Atri Rudra, Christopher Ré + Licensed under the BSD 3-Clause License + Source: https://github.com/Dao-AILab/flash-attention + + Optimization ideas such as tiling and recomputation were inspired by this work. + +5. AutoAWQ + Copyright (c) 2023 Casper Hansen + Licensed under the MIT License + Source: https://github.com/casper-hansen/AutoAWQ + + The design of the automodel was referenced from this project. + +6. llm.c + Copyright (c) 2023 Andrej Karpathy + Licensed under the MIT License + Source: https://github.com/karpathy/llm.c + + The design of end-to-end testing was referenced from this project. + +7. Tiny Shakespeare Dataset + Source: https://huggingface.co/datasets/karpathy/tiny_shakespeare + + This dataset is used to conduct convergence tests on mini models. + +For full license texts, please refer to the respective project repositories. diff --git a/benchmark/BENCHMARK_GUIDELINES.md b/benchmark/BENCHMARK_GUIDELINES.md new file mode 100755 index 0000000..9072234 --- /dev/null +++ b/benchmark/BENCHMARK_GUIDELINES.md @@ -0,0 +1,101 @@ +# Guideline for Adding Benchmark Scripts + +This document describes how to add new benchmark scripts to Liger-Kernel in line with the shared framework. + +## 1. Where and how to add a script + +- **Location**: `benchmark/scripts/` +- **Naming**: `benchmark_.py` (e.g. `benchmark_geglu.py`, `benchmark_swiglu.py`) + +## 2. Use shared infrastructure + +Do **not** hardcode batch size, sequence length, or model dimensions. Use: + +| Need | Use | +|------|-----| +| Model dimensions (hidden_size, vocab_size, etc.) | `benchmark_model_configs.py`: `ModelConfig`, `get_benchmark_model_config()` | +| Safe sweep config (seq_len or hidden_size) | `compute_seq_len_sweep_config()` (returns `SeqLenSweepConfig`) or `compute_hidden_size_sweep_config()` (returns `HiddenSizeSweepConfig`), with optional `estimate_kernel_peak_memory()` | +| Speed / memory measurement | `utils.py`: `run_speed_benchmark()`, `run_memory_benchmark()` | +| CLI (overwrite, model choice) | `utils.py`: `parse_benchmark_script_args()` (includes `--model`) | +| Running the grid and writing CSV | `utils.py`: `run_benchmarks()` | + +## 3. Script structure (three parts) + +### 3.1 Setup factory + +Define a single **setup function** that builds inputs and the layer (or callable) from `SingleBenchmarkRunInput`, so both speed and memory benchmarks reuse the same setup. + +- **Signature**: `_setup_(input: SingleBenchmarkRunInput) -> (tensors, layer_or_fn)` +- **Input**: `input.x` is the varying dimension (e.g. sequence length); `input.extra_benchmark_config` holds `bsz`, `hidden_size`, `dtype`, etc.; `input.kernel_provider` identifies the implementation variant (e.g. `"liger"`, `"huggingface"`, `"torch"`; values are kernel-specific). +- **Return**: Whatever the benchmark helpers need (e.g. `(x, layer)` for a single-tensor forward like GEGLU). + +Example (conceptually): + +```python +def _setup_geglu(input: SingleBenchmarkRunInput): + cfg = input.extra_benchmark_config + # Build config, create x tensor, instantiate LigerGEGLUMLP or LlamaMLP by provider + return x, layer +``` + +### 3.2 Speed and memory benchmark functions + +Each takes `SingleBenchmarkRunInput` and returns `SingleBenchmarkRunOutput` by calling the shared helpers. + +- **Speed**: `run_speed_benchmark(fwd_fn, mode, input_tensors, rep=...)` +- **Memory**: `run_memory_benchmark(fwd_fn, mode)` +- **Modes**: Use `["full", "forward", "backward"]` for both speed and memory for consistency. + +Example: + +```python +def bench_speed_geglu(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_geglu(input) + return run_speed_benchmark(lambda: layer(x), input.kernel_operation_mode, [x]) + +def bench_memory_geglu(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_geglu(input) + return run_memory_benchmark(lambda: layer(x), input.kernel_operation_mode) +``` + +For **scalar output** (e.g. loss) or **multiple outputs** (e.g. RoPE), use the appropriate helpers from `utils.py` if available (e.g. loss or multi-output variants), or implement custom measurement and still use the same setup factory and `run_benchmarks()`. + +### 3.3 `__main__`: model config, shape computation, run + +1. Parse args: `args = parse_benchmark_script_args()` and resolve `model = get_benchmark_model_config(args.model)`. +2. (Recommended) Measure peak memory with a small probe using the **highest-memory baseline** implementation (e.g. `"huggingface"` or `"torch"`): + - Define a `_probe()` function that creates tensors/layers, runs a forward pass, and returns the output tensor. `_probe()` owns setup; `estimate_kernel_peak_memory` handles memory-stat reset before the call, runs `.backward()`, and performs cleanup (gc + cache clear) afterward. + - Call `peak_bytes = estimate_kernel_peak_memory(probe_fn=_probe)`. +3. Compute sweep config (device memory is obtained internally by both helpers): + - **Sequence-length sweep** (e.g. GEGLU, SwiGLU): convert peak bytes to per-token (`kernel_bpt = peak_bytes // probe_seq_len`), then `config = compute_seq_len_sweep_config(model, kernel_bytes_per_token=kernel_bpt)`. The returned `SeqLenSweepConfig` has `batch_size` and `seq_len`. + - **Hidden-size sweep** (e.g. DyT): pass total peak bytes directly: `config = compute_hidden_size_sweep_config(model, kernel_peak_bytes=peak_bytes, bt=BT)`. The returned `HiddenSizeSweepConfig` has `bt` and `max_hidden_size`. +4. Build `x_values` from `config.seq_len` (seq_len sweep) or `config.max_hidden_size` (hidden_size sweep). +5. Build `extra_benchmark_configs` from `model` and config: + - Seq_len sweep: e.g. `bsz=config.batch_size`, `hidden_size=model.hidden_size`, `dtype=model.dtype`. + - Hidden_size sweep: e.g. `BT=config.bt`, `dtype=model.dtype`. +6. Call `run_benchmarks(..., kernel_operation_modes=["full", "forward", "backward"], ...)` for both speed and memory. + +## 4. CLI + +Scripts should support: + +- `--overwrite`: overwrite existing rows in the benchmark CSV. +- `--model`: model profile name from `MODEL_REGISTRY` (e.g. `llama_2_7b`, `llama_3_8b`). Default when not set is `DEFAULT_MODEL_CONFIG` (e.g. `llama_3_8b`). + +These are provided by `parse_benchmark_script_args()` in `utils.py`. + +## 5. Reference scripts + +- **Element-wise (single tensor in/out, seq_len sweep)**: `benchmark_geglu.py`, `benchmark_swiglu.py` — `compute_seq_len_sweep_config()`. +- **Element-wise (single tensor in/out, hidden_size sweep)**: `benchmark_dyt.py` — `compute_hidden_size_sweep_config()`. + +## 6. Checklist for a new script + +- [ ] Script under `benchmark/scripts/` named `benchmark_.py`. +- [ ] Single `_setup_(SingleBenchmarkRunInput)` used by both speed and memory. +- [ ] Speed/memory implemented via `run_speed_benchmark` / `run_memory_benchmark` (or the correct variant for loss / multi-output). +- [ ] `kernel_operation_modes=["full", "forward", "backward"]` for both speed and memory. +- [ ] No hardcoded batch size or sequence length; use `compute_seq_len_sweep_config()` or `compute_hidden_size_sweep_config()` (and optionally `estimate_kernel_peak_memory()`). +- [ ] Model dimensions and dtype from `ModelConfig` / `get_benchmark_model_config()` / `args.model`. +- [ ] CLI via `parse_benchmark_script_args()` (so `--model` and `--overwrite` work). +- [ ] Results written through `run_benchmarks()` so data goes to the shared CSV. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100755 index 0000000..02c883d --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,48 @@ +## Benchmarking Liger Kernels + +Follow these steps to benchmark and visualize kernel performance: + +1. Create a benchmark script + - Add your script under `benchmark/scripts/` + - Name it according to the kernel (e.g., `benchmark_.py`) + +2. Run the benchmark + - Results will be saved to `benchmark/data/all_benchmark_data.csv` + + Example: Benchmarking KTO Loss + ```bash + cd benchmark + python scripts/benchmark_kto_loss.py + ``` + +3. Visualize results + - Use the visualization script with optional modes: + + * To target specific mode(s), pass `--kernel-operation-mode` one or more values. + * If you omit `--kernel-operation-mode`, the script will: + - For `speed` metrics: generate plots for all available modes (forward/backward/full). + - For `memory` metrics: generate only the `full` plot. + + Examples: + 1. Specific modes (speed): + ```bash + python benchmarks_visualizer.py \ + --kernel-name kto_loss \ + --metric-name speed \ + --kernel-operation-mode forward backward + ``` + 2. All modes (speed): + ```bash + python benchmarks_visualizer.py \ + --kernel-name kto_loss \ + --metric-name speed + ``` + 3. Memory (always full): + ```bash + python benchmarks_visualizer.py \ + --kernel-name kto_loss \ + --metric-name memory + ``` + +4. View results + - Generated plots will be saved in `benchmark/visualizations/` \ No newline at end of file diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/benchmark/benchmarks_visualizer.py b/benchmark/benchmarks_visualizer.py new file mode 100755 index 0000000..e33d844 --- /dev/null +++ b/benchmark/benchmarks_visualizer.py @@ -0,0 +1,299 @@ +import json +import os +import sys + +from argparse import ArgumentParser +from dataclasses import dataclass + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +DATA_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "data/all_benchmark_data.csv")) +VISUALIZATIONS_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "visualizations/")) + + +@dataclass +class VisualizationsConfig: + """ + Configuration for the visualizations script. + + Args: + kernel_name (str): Kernel name to benchmark. (Will run `scripts/benchmark_{kernel_name}.py`) + metric_name (str): Metric name to visualize (speed/memory) + kernel_operation_mode (str): Kernel operation mode to visualize (forward/backward/full). Defaults to "full" + extra_config_filter (str, optional): A string to filter extra_benchmark_config. + Can be a substring to match or a 'key=value' pair (e.g., "'H': 4096"). + Defaults to None, which means the first available config will be used if multiple exist. + display (bool): Display the visualization. Defaults to False + overwrite (bool): Overwrite existing visualization, if none exist this flag has no effect as ones are always created and saved. Defaults to False + + """ + + kernel_name: str + metric_name: str + kernel_operation_mode: str = "full" + extra_config_filter: str | None = None + display: bool = False + overwrite: bool = False + + +def parse_args() -> VisualizationsConfig: + """Parse command line arguments into a configuration object. + + Returns: + VisualizationsConfig: Configuration object for the visualizations script. + """ + parser = ArgumentParser() + parser.add_argument("--kernel-name", type=str, required=True, help="Kernel name to benchmark") + parser.add_argument( + "--metric-name", + type=str, + required=True, + help="Metric name to visualize (speed/memory)", + ) + parser.add_argument( + "--kernel-operation-mode", + type=str, + nargs="*", + default=None, + help="Kernel operation modes to visualize (forward/backward/full). If not provided, generate for all available modes.", + ) + parser.add_argument( + "--extra-config-filter", + type=str, + default=None, + help="A string to filter extra_benchmark_config. " + "Can be a substring to match or a JSON-like 'key=value' pair (e.g., \"'H': 4096\" or \"H=4096\" for simple cases). " + "Defaults to None (first available config if multiple exist).", + ) + parser.add_argument("--display", action="store_true", help="Display the visualization") + parser.add_argument( + "--overwrite", + action="store_true", + help="Overwrite existing visualization, if none exist this flag has no effect as one are always created", + ) + + args = parser.parse_args() + return args + + +def load_data(config: VisualizationsConfig) -> pd.DataFrame: + """Loads the benchmark data from the CSV file and filters it based on the configuration. + + Args: + config (VisualizationsConfig): Configuration object for the visualizations script. + + Raises: + ValueError: If no data is found for the given filters. + + Returns: + pd.DataFrame: Filtered benchmark dataframe. + """ + df = pd.read_csv(DATA_PATH) + df["extra_benchmark_config"] = df["extra_benchmark_config_str"].apply(json.loads) + + base_filtered_df = df[ + (df["kernel_name"] == config.kernel_name) + & (df["metric_name"] == config.metric_name) + & (df["kernel_operation_mode"] == config.kernel_operation_mode) + ] + + if base_filtered_df.empty: + raise ValueError( + f"No data found for kernel_name='{config.kernel_name}', " + f"metric_name='{config.metric_name}', " + f"kernel_operation_mode='{config.kernel_operation_mode}'." + ) + + unique_extra_configs_str = base_filtered_df["extra_benchmark_config_str"].unique() + selected_extra_config_str = None + + if len(unique_extra_configs_str) == 0: + print( + "Warning: No extra_benchmark_config found for the initial filters. " + "Proceeding with all data from initial filter." + ) + return base_filtered_df + + if config.extra_config_filter: + matched_configs = [] + try: + if "=" in config.extra_config_filter: + key_filter, value_filter = config.extra_config_filter.split("=", 1) + for cfg_str in unique_extra_configs_str: + cfg_json = json.loads(cfg_str) + if str(cfg_json.get(key_filter.strip("'\" "))) == value_filter.strip("'\" "): + matched_configs.append(cfg_str) + if not matched_configs: + matched_configs = [ + cfg_str for cfg_str in unique_extra_configs_str if config.extra_config_filter in cfg_str + ] + except Exception as e: + print( + f"Note: Could not parse extra_config_filter '{config.extra_config_filter}' as key=value ({e}), using substring match." + ) + matched_configs = [cfg_str for cfg_str in unique_extra_configs_str if config.extra_config_filter in cfg_str] + + if matched_configs: + if len(matched_configs) > 1: + print( + f"Warning: Multiple extra_benchmark_configs match filter '{config.extra_config_filter}': {matched_configs}. " + f"Using the first one: {matched_configs[0]}" + ) + selected_extra_config_str = matched_configs[0] + else: + print( + f"Warning: No extra_benchmark_config matches filter '{config.extra_config_filter}'. " + f"Available configs for {config.kernel_name} ({config.metric_name}, {config.kernel_operation_mode}): {list(unique_extra_configs_str)}" + ) + if len(unique_extra_configs_str) > 0: + selected_extra_config_str = unique_extra_configs_str[0] + print(f"Defaulting to the first available extra_benchmark_config: {selected_extra_config_str}") + else: + raise ValueError("No extra_benchmark_config available to select after failed filter attempt.") + + elif len(unique_extra_configs_str) > 1: + selected_extra_config_str = unique_extra_configs_str[0] + print( + f"Warning: Multiple extra_benchmark_configs found for {config.kernel_name} ({config.metric_name}, {config.kernel_operation_mode})." + ) + print(f"Defaulting to use: {selected_extra_config_str}") + print(f"Available configs: {list(unique_extra_configs_str)}") + print( + "Use the --extra-config-filter argument to select a specific one " + "(e.g., --extra-config-filter \"'H': 4096\" or a substring like \"'seq_len': 512\")." + ) + elif len(unique_extra_configs_str) == 1: + selected_extra_config_str = unique_extra_configs_str[0] + print(f"Using unique extra_benchmark_config: {selected_extra_config_str}") + + if selected_extra_config_str: + final_filtered_df = base_filtered_df[ + base_filtered_df["extra_benchmark_config_str"] == selected_extra_config_str + ] + else: + print("Warning: Could not select an extra_benchmark_config. Using data from initial filter if any.") + final_filtered_df = base_filtered_df + + if final_filtered_df.empty: + raise ValueError( + f"No data found after attempting to filter by extra_benchmark_config. " + f"Selected/Defaulted extra_config_str: {selected_extra_config_str}" + if selected_extra_config_str + else "No specific extra_config was selected." + ) + + print( + f"Plotting data for extra_benchmark_config: {json.loads(selected_extra_config_str if selected_extra_config_str else '{}')}" + ) + return final_filtered_df + + +def plot_data(df: pd.DataFrame, config: VisualizationsConfig): + """Plots the benchmark data, saving the result if needed. + + Args: + df (pd.DataFrame): Filtered benchmark dataframe. + config (VisualizationsConfig): Configuration object for the visualizations script. + """ + for col in ["y_value_20", "y_value_50", "y_value_80"]: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + + xlabel = df["x_label"].iloc[0] + ylabel = f"{config.metric_name} ({df['metric_unit'].iloc[0]})" + # Sort by "kernel_provider" to ensure consistent color assignment + df = df.sort_values(by="kernel_provider") + + plt.figure(figsize=(10, 6)) + sns.set(style="whitegrid") + try: + ax = sns.lineplot( + data=df, + x="x_value", + y="y_value_50", + hue="kernel_provider", + marker="o", + palette="tab10", + errorbar=("ci", None), + ) + except Exception: + ax = sns.lineplot( + data=df, + x="x_value", + y="y_value_50", + hue="kernel_provider", + marker="o", + palette="tab10", + errorbar=None, + ) + + # Seaborn can't plot pre-computed error bars, so we need to do it manually + lines = ax.get_lines() + colors = [line.get_color() for line in lines] + + for (_, group_data), color in zip(df.groupby("kernel_provider"), colors): + # for i, row in group_data.iterrows(): + y_error_lower = group_data["y_value_50"] - group_data["y_value_20"] + y_error_upper = group_data["y_value_80"] - group_data["y_value_50"] + y_error = [y_error_lower, y_error_upper] + + plt.errorbar( + group_data["x_value"], + group_data["y_value_50"], + yerr=y_error, + fmt="o", + color=color, + capsize=5, + ) + plt.legend(title="Kernel Provider") + plt.xlabel(xlabel) + plt.ylabel(ylabel) + plt.tight_layout() + + out_path = os.path.join( + VISUALIZATIONS_PATH, + f"{config.kernel_name}_{config.metric_name}_{config.kernel_operation_mode}.png", + ) + + if config.display: + plt.show() + if config.overwrite or not os.path.exists( + out_path + ): # Save the plot if it doesn't exist or if we want to overwrite it + os.makedirs(VISUALIZATIONS_PATH, exist_ok=True) + plt.savefig(out_path) + plt.close() + + +def main(): + args = parse_args() + all_df = pd.read_csv(DATA_PATH) + all_df["extra_benchmark_config"] = all_df["extra_benchmark_config_str"].apply(json.loads) + + if args.metric_name == "memory": + modes = ["full"] + elif args.kernel_operation_mode: + modes = args.kernel_operation_mode + else: + filtered = all_df[(all_df["kernel_name"] == args.kernel_name) & (all_df["metric_name"] == args.metric_name)] + modes = filtered["kernel_operation_mode"].unique().tolist() + if not modes: + print(f"No data found for kernel '{args.kernel_name}' and metric '{args.metric_name}'.", file=sys.stderr) + sys.exit(1) + + for mode in modes: + config = VisualizationsConfig( + kernel_name=args.kernel_name, + metric_name=args.metric_name, + kernel_operation_mode=mode, + display=args.display, + overwrite=args.overwrite, + ) + df = load_data(config) + plot_data(df, config) + + +if __name__ == "__main__": + main() diff --git a/benchmark/data/all_benchmark_data.csv b/benchmark/data/all_benchmark_data.csv new file mode 100755 index 0000000..f63286a --- /dev/null +++ b/benchmark/data/all_benchmark_data.csv @@ -0,0 +1,1957 @@ +kernel_name,kernel_provider,kernel_operation_mode,metric_name,metric_unit,x_name,x_label,x_value,y_value_50,y_value_20,y_value_80,extra_benchmark_config_str,gpu_name,timestamp,liger_version +cross_entropy,liger,forward,speed,ms,V,vocab size,4096,0.5324159860610962,0.5291008353233337,0.53476482629776,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:39,0.2.1 +cross_entropy,liger,forward,speed,ms,V,vocab size,8192,0.8101439476013184,0.7565760016441345,0.9144319891929626,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:39,0.2.1 +cross_entropy,liger,forward,speed,ms,V,vocab size,16384,1.4320800304412842,1.4087040424346924,1.5254720449447632,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:39,0.2.1 +cross_entropy,liger,forward,speed,ms,V,vocab size,32768,2.8378241062164307,2.805759906768799,2.9447360038757324,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:39,0.2.1 +cross_entropy,liger,forward,speed,ms,V,vocab size,65536,6.805135726928711,6.790579319000244,6.98748779296875,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:39,0.2.1 +cross_entropy,liger,forward,speed,ms,V,vocab size,131072,15.009359359741211,15.00483226776123,15.045599937438965,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:39,0.2.1 +cross_entropy,huggingface,forward,speed,ms,V,vocab size,4096,0.8751360177993774,0.87330561876297,0.8773248195648193,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:40,0.2.1 +cross_entropy,huggingface,forward,speed,ms,V,vocab size,8192,1.188480019569397,1.1871488094329834,1.1901824474334717,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:40,0.2.1 +cross_entropy,huggingface,forward,speed,ms,V,vocab size,16384,1.9522240161895752,1.9451839923858643,1.962073564529419,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:40,0.2.1 +cross_entropy,huggingface,forward,speed,ms,V,vocab size,32768,5.316768169403076,5.314131259918213,5.319046497344971,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:40,0.2.1 +cross_entropy,huggingface,forward,speed,ms,V,vocab size,65536,10.615103721618652,10.607129096984863,10.61723518371582,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:40,0.2.1 +cross_entropy,huggingface,forward,speed,ms,V,vocab size,131072,20.72643280029297,20.72038459777832,20.758554458618164,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:40,0.2.1 +cross_entropy,liger,full,speed,ms,V,vocab size,4096,0.8637440204620361,0.8607680201530457,0.8670976161956787,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:41,0.2.1 +cross_entropy,liger,full,speed,ms,V,vocab size,8192,1.462272047996521,1.4576319456100464,1.4661248922348022,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:41,0.2.1 +cross_entropy,liger,full,speed,ms,V,vocab size,16384,2.7454559803009033,2.741612672805786,2.780428647994995,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:41,0.2.1 +cross_entropy,liger,full,speed,ms,V,vocab size,32768,5.403264045715332,5.398873329162598,5.4122114181518555,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:41,0.2.1 +cross_entropy,liger,full,speed,ms,V,vocab size,65536,11.925024032592773,11.919878005981445,11.92919635772705,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:41,0.2.1 +cross_entropy,liger,full,speed,ms,V,vocab size,131072,25.22287940979004,25.21867561340332,25.23493766784668,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:41,0.2.1 +cross_entropy,huggingface,full,speed,ms,V,vocab size,4096,2.2260000705718994,2.2239038944244385,2.2290303707122803,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,speed,ms,V,vocab size,8192,3.5976319313049316,3.595616102218628,3.6007039546966553,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,speed,ms,V,vocab size,16384,6.8023200035095215,6.795276641845703,6.806528091430664,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,speed,ms,V,vocab size,32768,15.486032485961914,15.483936309814453,15.48681640625,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,speed,ms,V,vocab size,65536,30.778079986572266,30.76335334777832,30.77827262878418,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,speed,ms,V,vocab size,131072,60.43830490112305,60.43830490112305,60.43830490112305,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,liger,full,memory,MB,V,vocab size,4096,256.32861328125,256.32861328125,256.32861328125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,liger,full,memory,MB,V,vocab size,8192,512.32861328125,512.32861328125,512.32861328125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,liger,full,memory,MB,V,vocab size,16384,1024.32861328125,1024.32861328125,1024.32861328125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,liger,full,memory,MB,V,vocab size,32768,2048.32861328125,2048.32861328125,2048.32861328125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,liger,full,memory,MB,V,vocab size,65536,4096.32861328125,4096.32861328125,4096.32861328125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,liger,full,memory,MB,V,vocab size,131072,8192.328125,8192.328125,8192.328125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,memory,MB,V,vocab size,4096,1280.1259765625,1280.1259765625,1280.1259765625,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,memory,MB,V,vocab size,8192,2560.1259765625,2560.1259765625,2560.1259765625,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,memory,MB,V,vocab size,16384,5120.1259765625,5120.1259765625,5120.1259765625,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,memory,MB,V,vocab size,32768,10240.1259765625,10240.1259765625,10240.1259765625,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,memory,MB,V,vocab size,65536,20480.125,20480.125,20480.125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +cross_entropy,huggingface,full,memory,MB,V,vocab size,131072,40960.125,40960.125,40960.125,"{""B"": 8, ""T"": 2048}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:42,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,1024,0.04262400045990944,0.04214400053024292,0.04428799822926521,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,2048,0.04668800160288811,0.04560000076889992,0.04825599864125252,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,4096,0.0493599995970726,0.048153601586818695,0.05084799975156784,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,8192,0.05558399856090546,0.054207999259233475,0.0568000003695488,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,16384,0.061503998935222626,0.06022400036454201,0.06260479986667633,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,32768,0.06518399715423584,0.06406400352716446,0.06634879857301712,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,65536,0.06779199838638306,0.06656000018119812,0.06905599683523178,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,131072,0.07091200351715088,0.06963200122117996,0.07225599884986877,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:53,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,1024,0.16672000288963318,0.1416832059621811,0.16777600347995758,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,2048,0.14406399428844452,0.1435839980840683,0.1446399986743927,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,4096,0.1539199948310852,0.15334400534629822,0.1546431928873062,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,8192,0.1627199947834015,0.16179199516773224,0.16357119381427765,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,16384,0.1666879951953888,0.16587519645690918,0.16772480309009552,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,32768,0.1687680035829544,0.16784639656543732,0.1697216033935547,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,65536,0.16918399930000305,0.1685439944267273,0.17001600563526154,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,131072,0.17027199268341064,0.16927999258041382,0.17123199999332428,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:31:56,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,1024,0.039712000638246536,0.03798399865627289,0.04079360142350197,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,2048,0.04652800038456917,0.045318398624658585,0.04755200073122978,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,4096,0.05462399870157242,0.05361919850111008,0.05580800026655197,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,8192,0.06015999987721443,0.059487998485565186,0.06102399900555611,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,16384,0.06412799656391144,0.06329599767923355,0.06508159637451172,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,32768,0.066880002617836,0.06583040207624435,0.06777600198984146,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,65536,0.06896000355482101,0.06785280257463455,0.07009919732809067,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,131072,0.06915199756622314,0.0682239979505539,0.06998399645090103,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:01,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,1024,0.44515201449394226,0.4440639913082123,0.4463231861591339,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,2048,0.4620960056781769,0.4610239863395691,0.46300798654556274,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,4096,0.49136000871658325,0.4905087947845459,0.49270400404930115,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,8192,0.5527999997138977,0.5520448088645935,0.5538623929023743,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,16384,0.6350079774856567,0.6340479850769043,0.6363840103149414,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,32768,0.7710559964179993,0.7691839933395386,0.7727680206298828,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,65536,1.002560019493103,1.0006400346755981,1.004467248916626,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,131072,1.4482879638671875,1.4459072351455688,1.4513407945632935,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:05,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,1024,0.4537919759750366,0.4517247974872589,0.46081918478012085,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,2048,0.47407999634742737,0.4729023873806,0.47523200511932373,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,4096,0.5310080051422119,0.5298879742622375,0.5320383906364441,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,8192,0.6528639793395996,0.6514303684234619,0.6546239852905273,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,16384,0.8056960105895996,0.8048319816589355,0.807424008846283,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,32768,0.954543948173523,0.9533119797706604,0.9559999704360962,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,65536,1.1960480213165283,1.1946111917495728,1.1982656717300415,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,131072,1.642624020576477,1.6409599781036377,1.6447807550430298,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:08,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,1024,0.3001280128955841,0.29503998160362244,0.30576640367507935,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,2048,0.297760009765625,0.2938239872455597,0.3054080009460449,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,4096,0.2991679906845093,0.2956480085849762,0.3070079982280731,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,8192,0.2961280047893524,0.2899264097213745,0.3029248118400574,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,16384,0.3465920090675354,0.34563198685646057,0.3476351797580719,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,32768,0.46585598587989807,0.4641471803188324,0.4674175977706909,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,65536,0.6924160122871399,0.6907200217247009,0.6938239932060242,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,131072,1.1352640390396118,1.1327999830245972,1.1376447677612305,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:13,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,1024,0.18961599469184875,0.1879040002822876,0.19174399971961975,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,2048,0.21296000480651855,0.2112639993429184,0.21513600647449493,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,4096,0.2367040067911148,0.23467519879341125,0.23888640105724335,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,8192,0.26335999369621277,0.26099199056625366,0.2656640112400055,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,16384,0.2850880026817322,0.28336000442504883,0.2869440019130707,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,32768,0.30460798740386963,0.3023360073566437,0.30684158205986023,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,65536,0.31569600105285645,0.3138048052787781,0.3180544078350067,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,liger,forward,speed,ms,V,embedding dimension,131072,0.31988799571990967,0.31808000802993774,0.3219392001628876,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:28,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,1024,0.7865599989891052,0.7846271991729736,0.7891008257865906,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,2048,0.8262079954147339,0.8236607909202576,0.8279871940612793,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,4096,0.8446240425109863,0.8429504036903381,0.8475391864776611,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,8192,0.8540480136871338,0.8518400192260742,0.8557760119438171,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,16384,0.857695996761322,0.8553280234336853,0.8595200181007385,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,32768,0.8596479892730713,0.8576639890670776,0.8618879914283752,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,65536,1.0087039470672607,0.8624832034111023,1.0126848220825195,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,huggingface,forward,speed,ms,V,embedding dimension,131072,0.8633919954299927,0.8609600067138672,0.8647680282592773,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:43,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,1024,0.2572160065174103,0.255840003490448,0.25833600759506226,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,2048,0.2817760109901428,0.2805440127849579,0.2831552028656006,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,4096,0.30182400345802307,0.3002175986766815,0.3032831847667694,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,8192,0.3126400113105774,0.3114303946495056,0.31427839398384094,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,16384,0.3190400004386902,0.31795841455459595,0.32016000151634216,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,32768,0.32419198751449585,0.32281601428985596,0.32559359073638916,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,65536,0.3238080143928528,0.32236799597740173,0.3250240087509155,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,torch_compile,forward,speed,ms,V,embedding dimension,131072,0.3256959915161133,0.32434558868408203,0.32689279317855835,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:32:58,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,1024,2.17740797996521,2.1755776405334473,2.180025577545166,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,2048,2.2861440181732178,2.284735918045044,2.2882239818573,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,4096,2.4825921058654785,2.48024320602417,2.484800100326538,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,8192,2.74452805519104,2.7430784702301025,2.7452287673950195,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,16384,3.1216320991516113,3.1202433109283447,3.125638484954834,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,32768,3.7801599502563477,3.774118423461914,3.7824511528015137,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,65536,4.991136074066162,4.9875006675720215,4.993491172790527,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,liger,full,speed,ms,V,embedding dimension,131072,7.383471965789795,7.377497673034668,7.386828899383545,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:13,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,1024,1.5774879455566406,1.5668543577194214,1.7933248281478882,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,2048,1.7074079513549805,1.7012799978256226,1.8109056949615479,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,4096,1.950543999671936,1.9466559886932373,1.9592640399932861,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,8192,2.404927968978882,2.400460720062256,2.4551360607147217,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,16384,3.119904041290283,3.1171774864196777,3.1267263889312744,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,32768,4.32857608795166,4.321491241455078,4.439519882202148,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,65536,5.065216064453125,5.059558391571045,5.115980625152588,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,huggingface,full,speed,ms,V,embedding dimension,131072,7.489376068115234,7.484294414520264,7.5203776359558105,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:28,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,1024,1.0930559635162354,1.0918079614639282,1.0945919752120972,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,2048,1.1930559873580933,1.191705584526062,1.1951104402542114,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,4096,1.3096319437026978,1.3073855638504028,1.3119615316390991,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,8192,1.4822720289230347,1.480512022972107,1.4839999675750732,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,16384,1.7870559692382812,1.7859647274017334,1.7892736196517944,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,32768,2.3838400840759277,2.381312131881714,2.3860929012298584,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,65536,3.7430078983306885,3.740166425704956,3.745452880859375,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,torch_compile,full,speed,ms,V,embedding dimension,131072,5.940896034240723,5.934713363647461,5.943462371826172,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:43,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,1024,12348.125,12348.125,12348.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,2048,12360.125,12360.125,12360.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,4096,12384.125,12384.125,12384.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,8192,12432.125,12432.125,12432.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,16384,12528.125,12528.125,12528.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,32768,12720.125,12720.125,12720.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,65536,13104.125,13104.125,13104.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,131072,13872.125,13872.125,13872.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:45,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,1024,12356.537109375,12356.537109375,12356.537109375,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,2048,12371.359375,12371.359375,12371.359375,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,4096,12401.40625,12401.40625,12401.40625,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,8192,12461.5,12461.5,12461.5,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,16384,12581.6875,12581.6875,12581.6875,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,32768,12773.6875,12773.6875,12773.6875,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,65536,13157.6875,13157.6875,13157.6875,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,131072,13925.6875,13925.6875,13925.6875,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:48,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,1024,12348.125,12348.125,12348.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,2048,12366.125,12366.125,12366.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,4096,12402.125,12402.125,12402.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,8192,12474.125,12474.125,12474.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,16384,12618.125,12618.125,12618.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,32768,12906.125,12906.125,12906.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,65536,13482.125,13482.125,13482.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,131072,14634.125,14634.125,14634.125,"{""B"": 32, ""T"": 512, ""D"": 768, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:33:52,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,1024,14346.125,14346.125,14346.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,2048,14410.125,14410.125,14410.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,4096,14538.125,14538.125,14538.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,8192,14794.125,14794.125,14794.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,16384,15306.125,15306.125,15306.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,32768,16330.125,16330.125,16330.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,65536,18378.125,18378.125,18378.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,liger,full,memory,MB,V,embedding dimension,131072,22474.125,22474.125,22474.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:04,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,1024,14388.130859375,14388.130859375,14388.130859375,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,2048,14468.154296875,14468.154296875,14468.154296875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,4096,14628.201171875,14628.201171875,14628.201171875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,8192,14948.294921875,14948.294921875,14948.294921875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,16384,15588.482421875,15588.482421875,15588.482421875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,32768,16612.482421875,16612.482421875,16612.482421875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,65536,18660.482421875,18660.482421875,18660.482421875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,huggingface,full,memory,MB,V,embedding dimension,131072,22756.482421875,22756.482421875,22756.482421875,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:17,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,1024,14346.125,14346.125,14346.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,2048,14442.125,14442.125,14442.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,4096,14634.125,14634.125,14634.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,8192,15018.125,15018.125,15018.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,16384,1536.125,1536.125,1536.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,32768,3072.125,3072.125,3072.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,65536,6144.125,6144.125,6144.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +embedding,torch_compile,full,memory,MB,V,embedding dimension,131072,12288.125,12288.125,12288.125,"{""B"": 8, ""T"": 2048, ""D"": 4096, ""dtype"": ""torch.float32""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:34:31,0.2.1 +fused_linear_cross_entropy,liger,forward,speed,ms,BT,B x T,4096,119.52153778076172,119.52153778076172,119.52153778076172,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:03,0.4.2 +fused_linear_cross_entropy,liger,forward,speed,ms,BT,B x T,8192,168.08563232421875,168.08563232421875,168.08563232421875,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:03,0.4.2 +fused_linear_cross_entropy,liger,forward,speed,ms,BT,B x T,16384,274.07342529296875,274.07342529296875,274.07342529296875,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:03,0.4.2 +fused_linear_cross_entropy,liger,forward,speed,ms,BT,B x T,32768,508.4652099609375,508.4652099609375,508.4652099609375,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:03,0.4.2 +fused_linear_cross_entropy,huggingface,forward,speed,ms,BT,B x T,4096,20.911680221557617,20.90903663635254,20.915321350097656,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:34,0.4.2 +fused_linear_cross_entropy,huggingface,forward,speed,ms,BT,B x T,8192,37.97203063964844,37.9546012878418,37.989463806152344,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:34,0.4.2 +fused_linear_cross_entropy,huggingface,forward,speed,ms,BT,B x T,16384,76.39142608642578,76.39142608642578,76.39142608642578,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:34,0.4.2 +fused_linear_cross_entropy,huggingface,forward,speed,ms,BT,B x T,32768,151.91404724121094,151.91404724121094,151.91404724121094,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:44:34,0.4.2 +fused_linear_cross_entropy,liger,full,speed,ms,BT,B x T,4096,121.43059539794922,121.43059539794922,121.43059539794922,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:11,0.4.2 +fused_linear_cross_entropy,liger,full,speed,ms,BT,B x T,8192,166.70867919921875,166.70867919921875,166.70867919921875,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:11,0.4.2 +fused_linear_cross_entropy,liger,full,speed,ms,BT,B x T,16384,277.1166687011719,277.1166687011719,277.1166687011719,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:11,0.4.2 +fused_linear_cross_entropy,liger,full,speed,ms,BT,B x T,32768,511.0638732910156,511.0638732910156,511.0638732910156,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:11,0.4.2 +fused_linear_cross_entropy,huggingface,full,speed,ms,BT,B x T,4096,55.96684646606445,55.96684646606445,55.96684646606445,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:46,0.4.2 +fused_linear_cross_entropy,huggingface,full,speed,ms,BT,B x T,8192,111.45471954345703,111.45471954345703,111.45471954345703,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:46,0.4.2 +fused_linear_cross_entropy,huggingface,full,speed,ms,BT,B x T,16384,220.7836151123047,220.7836151123047,220.7836151123047,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:46,0.4.2 +fused_linear_cross_entropy,huggingface,full,speed,ms,BT,B x T,32768,452.4712829589844,452.4712829589844,452.4712829589844,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:45:46,0.4.2 +fused_linear_cross_entropy,liger,full,memory,MB,BT,B x T,4096,4245.5478515625,4245.5478515625,4245.5478515625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:25,0.4.2 +fused_linear_cross_entropy,liger,full,memory,MB,BT,B x T,8192,4466.9697265625,4466.9697265625,4466.9697265625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:25,0.4.2 +fused_linear_cross_entropy,liger,full,memory,MB,BT,B x T,16384,4910.4384765625,4910.4384765625,4910.4384765625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:25,0.4.2 +fused_linear_cross_entropy,liger,full,memory,MB,BT,B x T,32768,5794.6259765625,5794.6259765625,5794.6259765625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:25,0.4.2 +fused_linear_cross_entropy,huggingface,full,memory,MB,BT,B x T,4096,6092.2822265625,6092.2822265625,6092.2822265625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:53,0.4.2 +fused_linear_cross_entropy,huggingface,full,memory,MB,BT,B x T,8192,9162.3134765625,9162.3134765625,9162.3134765625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:53,0.4.2 +fused_linear_cross_entropy,huggingface,full,memory,MB,BT,B x T,16384,15302.3759765625,15302.3759765625,15302.3759765625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:53,0.4.2 +fused_linear_cross_entropy,huggingface,full,memory,MB,BT,B x T,32768,27582.5,27582.5,27582.5,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-22 17:46:53,0.4.2 +geglu,liger,full,speed,ms,T,sequence length,1024,30.03536033630371,30.03536033630371,30.03536033630371,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:14,0.2.1 +geglu,liger,full,speed,ms,T,sequence length,2048,54.04060745239258,54.04060745239258,54.04060745239258,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:14,0.2.1 +geglu,liger,full,speed,ms,T,sequence length,4096,108.52435302734375,108.52435302734375,108.52435302734375,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:14,0.2.1 +geglu,liger,full,speed,ms,T,sequence length,8192,216.6227264404297,216.6227264404297,216.6227264404297,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:14,0.2.1 +geglu,huggingface,full,speed,ms,T,sequence length,1024,27.938560485839844,27.938560485839844,27.938560485839844,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:21,0.2.1 +geglu,huggingface,full,speed,ms,T,sequence length,2048,54.51279830932617,54.51279830932617,54.51279830932617,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:21,0.2.1 +geglu,huggingface,full,speed,ms,T,sequence length,4096,110.97718048095703,110.97718048095703,110.97718048095703,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:21,0.2.1 +geglu,huggingface,full,speed,ms,T,sequence length,8192,220.93954467773438,220.93954467773438,220.93954467773438,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:21,0.2.1 +geglu,liger,forward,speed,ms,T,sequence length,1024,9.280096054077148,9.280096054077148,9.280096054077148,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:26,0.2.1 +geglu,liger,forward,speed,ms,T,sequence length,2048,17.59040069580078,17.59040069580078,17.59040069580078,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:26,0.2.1 +geglu,liger,forward,speed,ms,T,sequence length,4096,36.18726348876953,36.18726348876953,36.18726348876953,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:26,0.2.1 +geglu,liger,forward,speed,ms,T,sequence length,8192,72.60655975341797,72.60655975341797,72.60655975341797,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:26,0.2.1 +geglu,huggingface,forward,speed,ms,T,sequence length,1024,9.257439613342285,9.257439613342285,9.257439613342285,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:31,0.2.1 +geglu,huggingface,forward,speed,ms,T,sequence length,2048,18.099519729614258,18.099519729614258,18.099519729614258,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:31,0.2.1 +geglu,huggingface,forward,speed,ms,T,sequence length,4096,36.37263870239258,36.37263870239258,36.37263870239258,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:31,0.2.1 +geglu,huggingface,forward,speed,ms,T,sequence length,8192,72.66553497314453,72.66553497314453,72.66553497314453,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:31,0.2.1 +geglu,liger,backward,speed,ms,T,sequence length,1024,18.088287353515625,18.088287353515625,18.088287353515625,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:37,0.2.1 +geglu,liger,backward,speed,ms,T,sequence length,2048,35.195518493652344,35.195518493652344,35.195518493652344,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:37,0.2.1 +geglu,liger,backward,speed,ms,T,sequence length,4096,70.51395416259766,70.51395416259766,70.51395416259766,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:37,0.2.1 +geglu,liger,backward,speed,ms,T,sequence length,8192,141.28550720214844,141.28550720214844,141.28550720214844,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:37,0.2.1 +geglu,huggingface,backward,speed,ms,T,sequence length,1024,18.521728515625,18.521728515625,18.521728515625,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:42,0.2.1 +geglu,huggingface,backward,speed,ms,T,sequence length,2048,36.045406341552734,36.045406341552734,36.045406341552734,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:42,0.2.1 +geglu,huggingface,backward,speed,ms,T,sequence length,4096,72.88412475585938,72.88412475585938,72.88412475585938,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:42,0.2.1 +geglu,huggingface,backward,speed,ms,T,sequence length,8192,144.2132110595703,144.2132110595703,144.2132110595703,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:42,0.2.1 +geglu,liger,full,memory,MB,T,sequence length,1024,1582.25,1582.25,1582.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:45,0.2.1 +geglu,liger,full,memory,MB,T,sequence length,2048,2546.25,2546.25,2546.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:45,0.2.1 +geglu,liger,full,memory,MB,T,sequence length,4096,4474.25,4474.25,4474.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:45,0.2.1 +geglu,liger,full,memory,MB,T,sequence length,8192,8330.25,8330.25,8330.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:45,0.2.1 +geglu,huggingface,full,memory,MB,T,sequence length,1024,1992.25,1992.25,1992.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:50,0.2.1 +geglu,huggingface,full,memory,MB,T,sequence length,2048,3452.25,3452.25,3452.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:50,0.2.1 +geglu,huggingface,full,memory,MB,T,sequence length,4096,6372.25,6372.25,6372.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:50,0.2.1 +geglu,huggingface,full,memory,MB,T,sequence length,8192,12212.25,12212.25,12212.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:50,0.2.1 +geglu,liger,forward,memory,MB,T,sequence length,1024,918.25,918.25,918.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:55,0.2.1 +geglu,liger,forward,memory,MB,T,sequence length,2048,1562.25,1562.25,1562.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:55,0.2.1 +geglu,liger,forward,memory,MB,T,sequence length,4096,2850.25,2850.25,2850.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:55,0.2.1 +geglu,liger,forward,memory,MB,T,sequence length,8192,5426.25,5426.25,5426.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:55,0.2.1 +geglu,huggingface,forward,memory,MB,T,sequence length,1024,1090.25,1090.25,1090.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:58,0.2.1 +geglu,huggingface,forward,memory,MB,T,sequence length,2048,1906.25,1906.25,1906.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:58,0.2.1 +geglu,huggingface,forward,memory,MB,T,sequence length,4096,3538.25,3538.25,3538.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:58,0.2.1 +geglu,huggingface,forward,memory,MB,T,sequence length,8192,6802.25,6802.25,6802.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:38:58,0.2.1 +geglu,liger,backward,memory,MB,T,sequence length,1024,1582.25,1582.25,1582.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:02,0.2.1 +geglu,liger,backward,memory,MB,T,sequence length,2048,2546.25,2546.25,2546.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:02,0.2.1 +geglu,liger,backward,memory,MB,T,sequence length,4096,4474.25,4474.25,4474.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:02,0.2.1 +geglu,liger,backward,memory,MB,T,sequence length,8192,8330.25,8330.25,8330.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:02,0.2.1 +geglu,huggingface,backward,memory,MB,T,sequence length,1024,1992.25,1992.25,1992.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:06,0.2.1 +geglu,huggingface,backward,memory,MB,T,sequence length,2048,3452.25,3452.25,3452.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:06,0.2.1 +geglu,huggingface,backward,memory,MB,T,sequence length,4096,6372.25,6372.25,6372.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:06,0.2.1 +geglu,huggingface,backward,memory,MB,T,sequence length,8192,12212.25,12212.25,12212.25,"{""bsz"": 8, ""hidden_size"": 4096, ""intermediate_size"": 11008, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:06,0.2.1 +layer_norm,liger,forward,speed,ms,N,hidden size,1024,0.030271999537944794,0.02921600081026554,0.03142400085926056,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:14,0.2.1 +layer_norm,liger,forward,speed,ms,N,hidden size,2048,0.04992000013589859,0.04912000149488449,0.050783999264240265,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:14,0.2.1 +layer_norm,liger,forward,speed,ms,N,hidden size,4096,0.08816000074148178,0.08739200234413147,0.08899199962615967,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:14,0.2.1 +layer_norm,liger,forward,speed,ms,N,hidden size,8192,0.16521599888801575,0.16435199975967407,0.16627199947834015,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:14,0.2.1 +layer_norm,liger,forward,speed,ms,N,hidden size,16384,0.32230401039123535,0.32070401310920715,0.32393598556518555,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:14,0.2.1 +layer_norm,huggingface,forward,speed,ms,N,hidden size,1024,0.034143999218940735,0.033376000821590424,0.03580800071358681,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:17,0.2.1 +layer_norm,huggingface,forward,speed,ms,N,hidden size,2048,0.05734400078654289,0.05615999922156334,0.05859199911355972,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:17,0.2.1 +layer_norm,huggingface,forward,speed,ms,N,hidden size,4096,0.1218239963054657,0.12054400146007538,0.12316799908876419,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:17,0.2.1 +layer_norm,huggingface,forward,speed,ms,N,hidden size,8192,0.25755199790000916,0.255840003490448,0.25939199328422546,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:17,0.2.1 +layer_norm,huggingface,forward,speed,ms,N,hidden size,16384,0.5066879987716675,0.5045183897018433,0.5089280009269714,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:17,0.2.1 +layer_norm,liger,full,speed,ms,N,hidden size,1024,0.28019198775291443,0.2780799865722656,0.284960001707077,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:19,0.2.1 +layer_norm,liger,full,speed,ms,N,hidden size,2048,0.27827200293540955,0.27638399600982666,0.2824704051017761,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:19,0.2.1 +layer_norm,liger,full,speed,ms,N,hidden size,4096,0.2847039997577667,0.27955201268196106,0.2908479869365692,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:19,0.2.1 +layer_norm,liger,full,speed,ms,N,hidden size,8192,0.4405759871006012,0.43780481815338135,0.4440320134162903,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:19,0.2.1 +layer_norm,liger,full,speed,ms,N,hidden size,16384,1.1488319635391235,1.1439871788024902,1.1527807712554932,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:19,0.2.1 +layer_norm,huggingface,full,speed,ms,N,hidden size,1024,0.11884800344705582,0.11750400066375732,0.12035199999809265,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,speed,ms,N,hidden size,2048,0.1966399997472763,0.19432319700717926,0.19888000190258026,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,speed,ms,N,hidden size,4096,0.43142399191856384,0.42931199073791504,0.4336639940738678,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,speed,ms,N,hidden size,8192,0.829584002494812,0.826918363571167,0.832857608795166,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,speed,ms,N,hidden size,16384,1.6212799549102783,1.6171647310256958,1.6246912479400635,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,liger,full,memory,MB,N,hidden size,1024,80.90625,80.90625,80.90625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,liger,full,memory,MB,N,hidden size,2048,161.78125,161.78125,161.78125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,liger,full,memory,MB,N,hidden size,4096,323.53125,323.53125,323.53125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,liger,full,memory,MB,N,hidden size,8192,647.03125,647.03125,647.03125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,liger,full,memory,MB,N,hidden size,16384,1294.03125,1294.03125,1294.03125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,memory,MB,N,hidden size,1024,80.0625,80.0625,80.0625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,memory,MB,N,hidden size,2048,160.09375,160.09375,160.09375,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,memory,MB,N,hidden size,4096,320.15625,320.15625,320.15625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,memory,MB,N,hidden size,8192,640.28125,640.28125,640.28125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +layer_norm,huggingface,full,memory,MB,N,hidden size,16384,1280.53125,1280.53125,1280.53125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:21,0.2.1 +rms_norm,liger,forward,speed,ms,H,hidden size,1024,0.01360000018030405,0.012864000163972378,0.01603199914097786,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:30,0.2.1 +rms_norm,liger,forward,speed,ms,H,hidden size,2048,0.019999999552965164,0.018624000251293182,0.02160000056028366,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:30,0.2.1 +rms_norm,liger,forward,speed,ms,H,hidden size,4096,0.031072000041604042,0.030047999694943428,0.031968001276254654,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:30,0.2.1 +rms_norm,liger,forward,speed,ms,H,hidden size,8192,0.0517439991235733,0.050624001771211624,0.05289600044488907,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:30,0.2.1 +rms_norm,liger,forward,speed,ms,H,hidden size,16384,0.0952640026807785,0.0942080020904541,0.09667199850082397,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:30,0.2.1 +rms_norm,liger,forward,speed,ms,H,hidden size,32768,0.18223999440670013,0.18035200238227844,0.18417279422283173,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:30,0.2.1 +rms_norm,huggingface,forward,speed,ms,H,hidden size,1024,0.07820799946784973,0.0777600035071373,0.0790719985961914,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:33,0.2.1 +rms_norm,huggingface,forward,speed,ms,H,hidden size,2048,0.13631999492645264,0.13555200397968292,0.13731199502944946,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:33,0.2.1 +rms_norm,huggingface,forward,speed,ms,H,hidden size,4096,0.27990400791168213,0.2789439857006073,0.28118398785591125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:33,0.2.1 +rms_norm,huggingface,forward,speed,ms,H,hidden size,8192,0.5190399885177612,0.5175359845161438,0.5209856033325195,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:33,0.2.1 +rms_norm,huggingface,forward,speed,ms,H,hidden size,16384,0.9856320023536682,0.9835839867591858,0.9876928329467773,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:33,0.2.1 +rms_norm,huggingface,forward,speed,ms,H,hidden size,32768,1.9190720319747925,1.917081594467163,1.921875238418579,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:33,0.2.1 +rms_norm,liger,full,speed,ms,H,hidden size,1024,0.28601598739624023,0.2837119996547699,0.29068800806999207,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:36,0.2.1 +rms_norm,liger,full,speed,ms,H,hidden size,2048,0.286624014377594,0.2845824062824249,0.2905920147895813,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:36,0.2.1 +rms_norm,liger,full,speed,ms,H,hidden size,4096,0.28830400109291077,0.28533118963241577,0.2935168147087097,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:36,0.2.1 +rms_norm,liger,full,speed,ms,H,hidden size,8192,0.29407998919487,0.289216011762619,0.3038719892501831,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:36,0.2.1 +rms_norm,liger,full,speed,ms,H,hidden size,16384,0.410863995552063,0.4088575839996338,0.41293439269065857,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:36,0.2.1 +rms_norm,liger,full,speed,ms,H,hidden size,32768,1.2316479682922363,1.228230357170105,1.235001564025879,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:36,0.2.1 +rms_norm,huggingface,full,speed,ms,H,hidden size,1024,0.3176960051059723,0.3147839903831482,0.32177281379699707,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:40,0.2.1 +rms_norm,huggingface,full,speed,ms,H,hidden size,2048,0.49038398265838623,0.4888896048069,0.4920639991760254,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:40,0.2.1 +rms_norm,huggingface,full,speed,ms,H,hidden size,4096,1.011423945426941,1.0089855194091797,1.013759970664978,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:40,0.2.1 +rms_norm,huggingface,full,speed,ms,H,hidden size,8192,1.8621759414672852,1.859769582748413,1.8646591901779175,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:40,0.2.1 +rms_norm,huggingface,full,speed,ms,H,hidden size,16384,3.5439999103546143,3.5410239696502686,3.547679901123047,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:40,0.2.1 +rms_norm,huggingface,full,speed,ms,H,hidden size,32768,6.910431861877441,6.907142639160156,6.914393901824951,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:40,0.2.1 +rms_norm,liger,backward,speed,ms,H,hidden size,1024,0.09372799843549728,0.09177599847316742,0.09763199836015701,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:43,0.2.1 +rms_norm,liger,backward,speed,ms,H,hidden size,2048,0.09030400216579437,0.08746880292892456,0.09398400038480759,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:43,0.2.1 +rms_norm,liger,backward,speed,ms,H,hidden size,4096,0.09913600236177444,0.09804800152778625,0.10039679706096649,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:43,0.2.1 +rms_norm,liger,backward,speed,ms,H,hidden size,8192,0.17801600694656372,0.1765120029449463,0.1793919950723648,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:43,0.2.1 +rms_norm,liger,backward,speed,ms,H,hidden size,16384,0.32051199674606323,0.3187839984893799,0.32230401039123535,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:43,0.2.1 +rms_norm,liger,backward,speed,ms,H,hidden size,32768,1.0562880039215088,1.053491234779358,1.059673547744751,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:43,0.2.1 +rms_norm,huggingface,backward,speed,ms,H,hidden size,1024,0.19577600061893463,0.19523200392723083,0.19631999731063843,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,backward,speed,ms,H,hidden size,2048,0.36188799142837524,0.3601599931716919,0.363647997379303,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,backward,speed,ms,H,hidden size,4096,0.7403839826583862,0.7381759881973267,0.7426176071166992,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,backward,speed,ms,H,hidden size,8192,1.3515520095825195,1.348736047744751,1.3550655841827393,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,backward,speed,ms,H,hidden size,16384,2.569632053375244,2.5663681030273438,2.5731201171875,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,backward,speed,ms,H,hidden size,32768,5.0147199630737305,5.011123180389404,5.0179901123046875,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,liger,full,memory,MB,H,hidden size,1024,36.02392578125,36.02392578125,36.02392578125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,liger,full,memory,MB,H,hidden size,2048,72.03955078125,72.03955078125,72.03955078125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,liger,full,memory,MB,H,hidden size,4096,144.07080078125,144.07080078125,144.07080078125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,liger,full,memory,MB,H,hidden size,8192,268.13330078125,268.13330078125,268.13330078125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,liger,full,memory,MB,H,hidden size,16384,432.25830078125,432.25830078125,432.25830078125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,liger,full,memory,MB,H,hidden size,32768,752.5087890625,752.5087890625,752.5087890625,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,full,memory,MB,H,hidden size,1024,80.01953125,80.01953125,80.01953125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,full,memory,MB,H,hidden size,2048,160.03125,160.03125,160.03125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,full,memory,MB,H,hidden size,4096,320.0546875,320.0546875,320.0546875,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,full,memory,MB,H,hidden size,8192,640.1015625,640.1015625,640.1015625,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,full,memory,MB,H,hidden size,16384,1280.1953125,1280.1953125,1280.1953125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rms_norm,huggingface,full,memory,MB,H,hidden size,32768,2560.3828125,2560.3828125,2560.3828125,"{""M"": 2048, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:46,0.2.1 +rope,liger,forward,speed,ms,H,hidden size,512,0.011359999887645245,0.01033599954098463,0.011455999687314034,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:55,0.2.1 +rope,liger,forward,speed,ms,H,hidden size,2048,0.020864000543951988,0.020447999238967896,0.02239999920129776,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:55,0.2.1 +rope,liger,forward,speed,ms,H,hidden size,8192,0.059487998485565186,0.05830400064587593,0.06060799956321716,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:55,0.2.1 +rope,huggingface,forward,speed,ms,H,hidden size,512,0.07968000322580338,0.07923199981451035,0.10408961027860641,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:56,0.2.1 +rope,huggingface,forward,speed,ms,H,hidden size,2048,0.1570879966020584,0.15651200711727142,0.15785600244998932,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:56,0.2.1 +rope,huggingface,forward,speed,ms,H,hidden size,8192,0.5167999863624573,0.5161600112915039,0.5176640152931213,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:56,0.2.1 +rope,liger,backward,speed,ms,H,hidden size,512,0.12227199971675873,0.05539200082421303,0.1699904054403305,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:57,0.2.1 +rope,liger,backward,speed,ms,H,hidden size,2048,0.12337599694728851,0.11945600062608719,0.15338242053985596,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:57,0.2.1 +rope,liger,backward,speed,ms,H,hidden size,8192,0.12812800705432892,0.11593600362539291,0.1985855996608734,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:57,0.2.1 +rope,huggingface,backward,speed,ms,H,hidden size,512,0.2648000121116638,0.2489279955625534,0.3578239977359772,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:59,0.2.1 +rope,huggingface,backward,speed,ms,H,hidden size,2048,0.2536320090293884,0.24692480266094208,0.31929606199264526,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:59,0.2.1 +rope,huggingface,backward,speed,ms,H,hidden size,8192,0.621504008769989,0.6208000183105469,0.6223679780960083,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:39:59,0.2.1 +rope,liger,full,speed,ms,H,hidden size,512,0.27401599287986755,0.26447999477386475,0.3555007874965668,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:00,0.2.1 +rope,liger,full,speed,ms,H,hidden size,2048,0.2815040051937103,0.26904961466789246,0.3562496304512024,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:00,0.2.1 +rope,liger,full,speed,ms,H,hidden size,8192,0.2759679853916168,0.267244815826416,0.3601728081703186,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:00,0.2.1 +rope,huggingface,full,speed,ms,H,hidden size,512,0.5160639882087708,0.5028480291366577,0.6553279757499695,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,huggingface,full,speed,ms,H,hidden size,2048,0.5289119482040405,0.510598361492157,0.7208256721496582,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,huggingface,full,speed,ms,H,hidden size,8192,1.1329920291900635,1.1318720579147339,1.1339199542999268,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,liger,full,memory,MB,H,hidden size,512,13.26611328125,13.26611328125,13.26611328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,liger,full,memory,MB,H,hidden size,2048,28.64111328125,28.64111328125,28.64111328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,liger,full,memory,MB,H,hidden size,8192,90.14111328125,90.14111328125,90.14111328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,huggingface,full,memory,MB,H,hidden size,512,22.26611328125,22.26611328125,22.26611328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,huggingface,full,memory,MB,H,hidden size,2048,64.64111328125,64.64111328125,64.64111328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,huggingface,full,memory,MB,H,hidden size,8192,234.14111328125,234.14111328125,234.14111328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:01,0.2.1 +rope,liger,forward,speed,ms,T,sequence length,1024,0.034432001411914825,0.03340800106525421,0.03545600175857544,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:04,0.2.1 +rope,liger,forward,speed,ms,T,sequence length,2048,0.058880001306533813,0.0578560009598732,0.059859201312065125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:04,0.2.1 +rope,liger,forward,speed,ms,T,sequence length,4096,0.10899200290441513,0.10784000158309937,0.1101439967751503,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:04,0.2.1 +rope,liger,forward,speed,ms,T,sequence length,8192,0.20927999913692474,0.20796799659729004,0.21059200167655945,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:04,0.2.1 +rope,liger,forward,speed,ms,T,sequence length,16384,0.4105280041694641,0.4089151918888092,0.41203200817108154,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:04,0.2.1 +rope,huggingface,forward,speed,ms,T,sequence length,1024,0.2808319926261902,0.28019198775291443,0.28160640597343445,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:06,0.2.1 +rope,huggingface,forward,speed,ms,T,sequence length,2048,0.5160959959030151,0.5155072212219238,0.5169280171394348,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:06,0.2.1 +rope,huggingface,forward,speed,ms,T,sequence length,4096,0.9947839975357056,0.9939200282096863,0.9956799745559692,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:06,0.2.1 +rope,huggingface,forward,speed,ms,T,sequence length,8192,1.9332640171051025,1.9323519468307495,1.9344960451126099,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:06,0.2.1 +rope,huggingface,forward,speed,ms,T,sequence length,16384,3.8169920444488525,3.815808057785034,3.8180160522460938,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:06,0.2.1 +rope,liger,backward,speed,ms,T,sequence length,1024,0.1260479986667633,0.12014079838991165,0.143449604511261,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:08,0.2.1 +rope,liger,backward,speed,ms,T,sequence length,2048,0.11606399714946747,0.11021439731121063,0.12432000041007996,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:08,0.2.1 +rope,liger,backward,speed,ms,T,sequence length,4096,0.12409599870443344,0.11817599833011627,0.1313920021057129,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:08,0.2.1 +rope,liger,backward,speed,ms,T,sequence length,8192,0.21004800498485565,0.20867200195789337,0.21164800226688385,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:08,0.2.1 +rope,liger,backward,speed,ms,T,sequence length,16384,0.4102399945259094,0.40871042013168335,0.4119040071964264,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:08,0.2.1 +rope,huggingface,backward,speed,ms,T,sequence length,1024,0.3304319977760315,0.3296447992324829,0.3314239978790283,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:10,0.2.1 +rope,huggingface,backward,speed,ms,T,sequence length,2048,0.6213759779930115,0.6205440163612366,0.6223359704017639,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:10,0.2.1 +rope,huggingface,backward,speed,ms,T,sequence length,4096,1.1872799396514893,1.1858432292938232,1.1886080503463745,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:10,0.2.1 +rope,huggingface,backward,speed,ms,T,sequence length,8192,2.321280002593994,2.318873643875122,2.324160099029541,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:10,0.2.1 +rope,huggingface,backward,speed,ms,T,sequence length,16384,4.557248115539551,4.550220966339111,4.560742378234863,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:10,0.2.1 +rope,liger,full,speed,ms,T,sequence length,1024,0.2682560086250305,0.2641535997390747,0.2762559950351715,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:12,0.2.1 +rope,liger,full,speed,ms,T,sequence length,2048,0.2654559910297394,0.26105600595474243,0.2746559977531433,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:12,0.2.1 +rope,liger,full,speed,ms,T,sequence length,4096,0.2650560140609741,0.2608831822872162,0.2715519964694977,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:12,0.2.1 +rope,liger,full,speed,ms,T,sequence length,8192,0.4158720076084137,0.41413119435310364,0.4178048074245453,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:12,0.2.1 +rope,liger,full,speed,ms,T,sequence length,16384,0.8167039752006531,0.8143680095672607,0.8189184069633484,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:12,0.2.1 +rope,huggingface,full,speed,ms,T,sequence length,1024,0.6059200167655945,0.6047679781913757,0.6072319746017456,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,huggingface,full,speed,ms,T,sequence length,2048,1.1326719522476196,1.1318080425262451,1.133631944656372,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,huggingface,full,speed,ms,T,sequence length,4096,2.176192045211792,2.175136089324951,2.177433729171753,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,huggingface,full,speed,ms,T,sequence length,8192,4.248256206512451,4.246367931365967,4.2566399574279785,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,huggingface,full,speed,ms,T,sequence length,16384,8.365951538085938,8.36348819732666,8.380928039550781,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,liger,full,memory,MB,T,sequence length,1024,49.13330078125,49.13330078125,49.13330078125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,liger,full,memory,MB,T,sequence length,2048,90.14111328125,90.14111328125,90.14111328125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,liger,full,memory,MB,T,sequence length,4096,172.15673828125,172.15673828125,172.15673828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,liger,full,memory,MB,T,sequence length,8192,336.18798828125,336.18798828125,336.18798828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,liger,full,memory,MB,T,sequence length,16384,664.25048828125,664.25048828125,664.25048828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:14,0.2.1 +rope,huggingface,full,memory,MB,T,sequence length,1024,121.13330078125,121.13330078125,121.13330078125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:15,0.2.1 +rope,huggingface,full,memory,MB,T,sequence length,2048,234.14111328125,234.14111328125,234.14111328125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:15,0.2.1 +rope,huggingface,full,memory,MB,T,sequence length,4096,460.15673828125,460.15673828125,460.15673828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:15,0.2.1 +rope,huggingface,full,memory,MB,T,sequence length,8192,912.18798828125,912.18798828125,912.18798828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:15,0.2.1 +rope,huggingface,full,memory,MB,T,sequence length,16384,1816.25048828125,1816.25048828125,1816.25048828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:15,0.2.1 +swiglu,liger,forward,speed,ms,T,sequence length,1024,5.06441593170166,5.06441593170166,5.06441593170166,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:24,0.2.1 +swiglu,liger,forward,speed,ms,T,sequence length,2048,10.075455665588379,10.075455665588379,10.075455665588379,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:24,0.2.1 +swiglu,liger,forward,speed,ms,T,sequence length,4096,18.001951217651367,18.001951217651367,18.001951217651367,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:24,0.2.1 +swiglu,liger,forward,speed,ms,T,sequence length,8192,35.930015563964844,35.930015563964844,35.930015563964844,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:24,0.2.1 +swiglu,huggingface,forward,speed,ms,T,sequence length,1024,4.582320213317871,4.5821757316589355,4.582464218139648,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:28,0.2.1 +swiglu,huggingface,forward,speed,ms,T,sequence length,2048,9.252832412719727,9.252832412719727,9.252832412719727,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:28,0.2.1 +swiglu,huggingface,forward,speed,ms,T,sequence length,4096,18.160255432128906,18.160255432128906,18.160255432128906,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:28,0.2.1 +swiglu,huggingface,forward,speed,ms,T,sequence length,8192,36.2911376953125,36.2911376953125,36.2911376953125,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:28,0.2.1 +swiglu,liger,full,memory,MB,T,sequence length,1024,1100.25,1100.25,1100.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:32,0.2.1 +swiglu,liger,full,memory,MB,T,sequence length,2048,1582.25,1582.25,1582.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:32,0.2.1 +swiglu,liger,full,memory,MB,T,sequence length,4096,2546.25,2546.25,2546.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:32,0.2.1 +swiglu,liger,full,memory,MB,T,sequence length,8192,4474.25,4474.25,4474.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:32,0.2.1 +swiglu,huggingface,full,memory,MB,T,sequence length,1024,1294.25,1294.25,1294.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:36,0.2.1 +swiglu,huggingface,full,memory,MB,T,sequence length,2048,1992.25,1992.25,1992.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:36,0.2.1 +swiglu,huggingface,full,memory,MB,T,sequence length,4096,3452.25,3452.25,3452.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:36,0.2.1 +swiglu,huggingface,full,memory,MB,T,sequence length,8192,6372.25,6372.25,6372.25,"{""B"": 4, ""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""intermediate_size"": 11008, ""hidden_act"": ""silu""}",NVIDIA A100-SXM4-80GB,2024-09-03 15:40:36,0.2.1 +kl_div,liger,full,memory,MB,V,vocab size,4096,1536.0009765625,1536.0009765625,1536.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:40,0.2.1 +kl_div,liger,full,memory,MB,V,vocab size,8192,3072.0009765625,3072.0009765625,3072.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:40,0.2.1 +kl_div,liger,full,memory,MB,V,vocab size,16384,6144.0009765625,6144.0009765625,6144.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:40,0.2.1 +kl_div,liger,full,memory,MB,V,vocab size,32768,12288.0009765625,12288.0009765625,12288.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:40,0.2.1 +kl_div,liger,full,memory,MB,V,vocab size,65536,24576.0,24576.0,24576.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:40,0.2.1 +kl_div,liger,full,memory,MB,V,vocab size,131072,49152.0,49152.0,49152.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:40,0.2.1 +kl_div,torch,full,memory,MB,V,vocab size,4096,1792.0,1792.0,1792.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:41,0.2.1 +kl_div,torch,full,memory,MB,V,vocab size,8192,3584.0,3584.0,3584.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:41,0.2.1 +kl_div,torch,full,memory,MB,V,vocab size,16384,7168.0,7168.0,7168.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:41,0.2.1 +kl_div,torch,full,memory,MB,V,vocab size,32768,14336.0,14336.0,14336.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:41,0.2.1 +kl_div,torch,full,memory,MB,V,vocab size,65536,28672.0,28672.0,28672.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:41,0.2.1 +kl_div,torch,full,memory,MB,V,vocab size,131072,57344.0,57344.0,57344.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:41,0.2.1 +kl_div,liger,forward,speed,ms,V,vocab size,4096,0.30640000104904175,0.30563199520111084,0.30745598673820496,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:43,0.2.1 +kl_div,liger,forward,speed,ms,V,vocab size,8192,0.5763360261917114,0.5754943490028381,0.5773376226425171,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:43,0.2.1 +kl_div,liger,forward,speed,ms,V,vocab size,16384,1.1176480054855347,1.1165119409561157,1.1186367273330688,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:43,0.2.1 +kl_div,liger,forward,speed,ms,V,vocab size,32768,2.1987199783325195,2.1970815658569336,2.200934410095215,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:43,0.2.1 +kl_div,liger,forward,speed,ms,V,vocab size,65536,4.356672286987305,4.355186939239502,4.358956813812256,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:43,0.2.1 +kl_div,liger,forward,speed,ms,V,vocab size,131072,8.697919845581055,8.690688133239746,8.703583717346191,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:43,0.2.1 +kl_div,torch,forward,speed,ms,V,vocab size,4096,1.3298559188842773,1.3287359476089478,1.331385612487793,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:45,0.2.1 +kl_div,torch,forward,speed,ms,V,vocab size,8192,2.594543933868408,2.592736005783081,2.596640110015869,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:45,0.2.1 +kl_div,torch,forward,speed,ms,V,vocab size,16384,5.13375997543335,5.1324286460876465,5.1364288330078125,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:45,0.2.1 +kl_div,torch,forward,speed,ms,V,vocab size,32768,10.225567817687988,10.225190162658691,10.227231979370117,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:45,0.2.1 +kl_div,torch,forward,speed,ms,V,vocab size,65536,20.412960052490234,20.411020278930664,20.415000915527344,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:45,0.2.1 +kl_div,torch,forward,speed,ms,V,vocab size,131072,40.818641662597656,40.816402435302734,40.82087707519531,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:45,0.2.1 +kl_div,liger,full,speed,ms,V,vocab size,4096,2.040031909942627,1.9614335298538208,2.192307233810425,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:46,0.2.1 +kl_div,liger,full,speed,ms,V,vocab size,8192,3.866431951522827,3.7955007553100586,3.8693249225616455,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:46,0.2.1 +kl_div,liger,full,speed,ms,V,vocab size,16384,7.261951923370361,7.255136013031006,7.281760215759277,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:46,0.2.1 +kl_div,liger,full,speed,ms,V,vocab size,32768,15.092127799987793,15.07801628112793,15.09660816192627,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:46,0.2.1 +kl_div,liger,full,speed,ms,V,vocab size,65536,29.921375274658203,29.914867401123047,29.921951293945312,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:46,0.2.1 +kl_div,liger,full,speed,ms,V,vocab size,131072,59.70220947265625,59.70220947265625,59.70220947265625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:46,0.2.1 +kl_div,torch,full,speed,ms,V,vocab size,4096,2.8552000522613525,2.852755069732666,2.856454372406006,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:48,0.2.1 +kl_div,torch,full,speed,ms,V,vocab size,8192,5.593632221221924,5.590988636016846,5.594636917114258,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:48,0.2.1 +kl_div,torch,full,speed,ms,V,vocab size,16384,11.124671936035156,11.122162818908691,11.125061988830566,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:48,0.2.1 +kl_div,torch,full,speed,ms,V,vocab size,32768,23.052032470703125,23.050334930419922,23.052589416503906,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:48,0.2.1 +kl_div,torch,full,speed,ms,V,vocab size,65536,46.063167572021484,46.05990219116211,46.06643295288086,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:48,0.2.1 +kl_div,torch,full,speed,ms,V,vocab size,131072,92.06393432617188,92.06393432617188,92.06393432617188,"{""B"": 8, ""T"": 2048}",NVIDIA H100 PCIe,2024-09-04 12:59:48,0.2.1 +jsd,liger,full,memory,MB,V,vocab size,4096,768.0029296875,768.0029296875,768.0029296875,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:31,0.3.1 +jsd,liger,full,memory,MB,V,vocab size,8192,1536.0029296875,1536.0029296875,1536.0029296875,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:31,0.3.1 +jsd,liger,full,memory,MB,V,vocab size,16384,3072.0048828125,3072.0048828125,3072.0048828125,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:31,0.3.1 +jsd,liger,full,memory,MB,V,vocab size,32768,6144.0087890625,6144.0087890625,6144.0087890625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:31,0.3.1 +jsd,liger,full,memory,MB,V,vocab size,65536,12288.0166015625,12288.0166015625,12288.0166015625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:31,0.3.1 +jsd,liger,full,memory,MB,V,vocab size,131072,24576.015625,24576.015625,24576.015625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:31,0.3.1 +jsd,torch,full,memory,MB,V,vocab size,4096,1664.0009765625,1664.0009765625,1664.0009765625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:33,0.3.1 +jsd,torch,full,memory,MB,V,vocab size,8192,3328.0009765625,3328.0009765625,3328.0009765625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:33,0.3.1 +jsd,torch,full,memory,MB,V,vocab size,16384,6656.0009765625,6656.0009765625,6656.0009765625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:33,0.3.1 +jsd,torch,full,memory,MB,V,vocab size,32768,13312.0009765625,13312.0009765625,13312.0009765625,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:33,0.3.1 +jsd,torch,full,memory,MB,V,vocab size,65536,26624.0,26624.0,26624.0,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:33,0.3.1 +jsd,torch,full,memory,MB,V,vocab size,131072,53248.0,53248.0,53248.0,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:33,0.3.1 +jsd,liger,forward,speed,ms,V,vocab size,4096,0.4651840031147003,0.4636736214160919,0.4659839868545532,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:37,0.3.1 +jsd,liger,forward,speed,ms,V,vocab size,8192,0.927888035774231,0.926751971244812,0.92952960729599,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:37,0.3.1 +jsd,liger,forward,speed,ms,V,vocab size,16384,10.96003246307373,10.942886352539062,10.970770835876465,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:37,0.3.1 +jsd,liger,forward,speed,ms,V,vocab size,32768,22.405792236328125,22.390380859375,22.41998863220215,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:37,0.3.1 +jsd,liger,forward,speed,ms,V,vocab size,65536,43.49095916748047,43.47438049316406,43.50754165649414,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:37,0.3.1 +jsd,liger,forward,speed,ms,V,vocab size,131072,87.0363540649414,87.0363540649414,87.0363540649414,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:37,0.3.1 +jsd,torch,forward,speed,ms,V,vocab size,4096,2.4744958877563477,2.4725184440612793,2.4764864444732666,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:38,0.3.1 +jsd,torch,forward,speed,ms,V,vocab size,8192,4.8528642654418945,4.851238250732422,4.854745864868164,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:38,0.3.1 +jsd,torch,forward,speed,ms,V,vocab size,16384,9.532496452331543,9.528634071350098,9.535890579223633,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:38,0.3.1 +jsd,torch,forward,speed,ms,V,vocab size,32768,18.91379165649414,18.911853790283203,18.919116973876953,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:38,0.3.1 +jsd,torch,forward,speed,ms,V,vocab size,65536,37.70152282714844,37.70074462890625,37.70229721069336,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:38,0.3.1 +jsd,torch,forward,speed,ms,V,vocab size,131072,75.37680053710938,75.37680053710938,75.37680053710938,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:38,0.3.1 +jsd,liger,full,speed,ms,V,vocab size,4096,1.2074079513549805,1.1739968061447144,1.2760319709777832,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:40,0.3.1 +jsd,liger,full,speed,ms,V,vocab size,8192,2.091792106628418,2.0771327018737793,2.106553554534912,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:40,0.3.1 +jsd,liger,full,speed,ms,V,vocab size,16384,12.928031921386719,12.8988676071167,12.936230659484863,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:40,0.3.1 +jsd,liger,full,speed,ms,V,vocab size,32768,26.55548858642578,26.550823211669922,26.570655822753906,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:40,0.3.1 +jsd,liger,full,speed,ms,V,vocab size,65536,51.6833610534668,51.6833610534668,51.6833610534668,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:40,0.3.1 +jsd,liger,full,speed,ms,V,vocab size,131072,103.12793731689453,103.12793731689453,103.12793731689453,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:40,0.3.1 +jsd,torch,full,speed,ms,V,vocab size,4096,5.397359848022461,5.392876625061035,5.39998722076416,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:43,0.3.1 +jsd,torch,full,speed,ms,V,vocab size,8192,10.60153579711914,10.597900390625,10.60470962524414,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:43,0.3.1 +jsd,torch,full,speed,ms,V,vocab size,16384,20.9442081451416,20.94247055053711,20.9469051361084,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:43,0.3.1 +jsd,torch,full,speed,ms,V,vocab size,32768,42.113216400146484,42.113216400146484,42.113216400146484,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:43,0.3.1 +jsd,torch,full,speed,ms,V,vocab size,65536,83.9959716796875,83.9959716796875,83.9959716796875,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:43,0.3.1 +jsd,torch,full,speed,ms,V,vocab size,131072,167.94175720214844,167.94175720214844,167.94175720214844,"{""B"": 4, ""T"": 2048}",NVIDIA H100 PCIe,2024-10-02 16:21:43,0.3.1 +fused_linear_jsd,liger,forward,speed,ms,BT,B x T,1024,110.02185821533203,110.02185821533203,110.02185821533203,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:18,0.3.1 +fused_linear_jsd,liger,forward,speed,ms,BT,B x T,2048,124.14070129394531,124.14070129394531,124.14070129394531,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:18,0.3.1 +fused_linear_jsd,liger,forward,speed,ms,BT,B x T,4096,143.15420532226562,143.15420532226562,143.15420532226562,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:18,0.3.1 +fused_linear_jsd,liger,forward,speed,ms,BT,B x T,8192,180.90406799316406,180.90406799316406,180.90406799316406,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:18,0.3.1 +fused_linear_jsd,torch,forward,speed,ms,BT,B x T,1024,9.556896209716797,9.550745964050293,9.576268196105957,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:20,0.3.1 +fused_linear_jsd,torch,forward,speed,ms,BT,B x T,2048,18.73731231689453,18.732704162597656,18.737701416015625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:20,0.3.1 +fused_linear_jsd,torch,forward,speed,ms,BT,B x T,4096,37.830482482910156,37.80821990966797,37.85274124145508,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:20,0.3.1 +fused_linear_jsd,torch,forward,speed,ms,BT,B x T,8192,75.15289306640625,75.15289306640625,75.15289306640625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:20,0.3.1 +fused_linear_jsd,liger,full,speed,ms,BT,B x T,1024,111.16019439697266,111.16019439697266,111.16019439697266,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:24,0.3.1 +fused_linear_jsd,liger,full,speed,ms,BT,B x T,2048,125.6825942993164,125.6825942993164,125.6825942993164,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:24,0.3.1 +fused_linear_jsd,liger,full,speed,ms,BT,B x T,4096,144.00784301757812,144.00784301757812,144.00784301757812,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:24,0.3.1 +fused_linear_jsd,liger,full,speed,ms,BT,B x T,8192,182.5832977294922,182.5832977294922,182.5832977294922,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:24,0.3.1 +fused_linear_jsd,torch,full,speed,ms,BT,B x T,1024,25.977184295654297,25.968351364135742,25.989356994628906,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:27,0.3.1 +fused_linear_jsd,torch,full,speed,ms,BT,B x T,2048,49.48417663574219,49.47330093383789,49.495052337646484,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:27,0.3.1 +fused_linear_jsd,torch,full,speed,ms,BT,B x T,4096,98.31510162353516,98.31510162353516,98.31510162353516,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:27,0.3.1 +fused_linear_jsd,torch,full,speed,ms,BT,B x T,8192,195.29539489746094,195.29539489746094,195.29539489746094,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:27,0.3.1 +fused_linear_jsd,liger,full,memory,MB,BT,B x T,1024,4652.48486328125,4652.48486328125,4652.48486328125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:33,0.3.1 +fused_linear_jsd,liger,full,memory,MB,BT,B x T,2048,5231.93798828125,5231.93798828125,5231.93798828125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:33,0.3.1 +fused_linear_jsd,liger,full,memory,MB,BT,B x T,4096,6391.87548828125,6391.87548828125,6391.87548828125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:33,0.3.1 +fused_linear_jsd,liger,full,memory,MB,BT,B x T,8192,8711.75,8711.75,8711.75,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:33,0.3.1 +fused_linear_jsd,torch,full,memory,MB,BT,B x T,1024,10609.005859375,10609.005859375,10609.005859375,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:35,0.3.1 +fused_linear_jsd,torch,full,memory,MB,BT,B x T,2048,17146.009765625,17146.009765625,17146.009765625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:35,0.3.1 +fused_linear_jsd,torch,full,memory,MB,BT,B x T,4096,30220.017578125,30220.017578125,30220.017578125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:35,0.3.1 +fused_linear_jsd,torch,full,memory,MB,BT,B x T,8192,56368.015625,56368.015625,56368.015625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H100 80GB HBM3,2024-10-09 12:29:35,0.3.1 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,2,116.00621032714844,116.00621032714844,116.00621032714844,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:05,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,4,230.83609008789062,230.83609008789062,230.83609008789062,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:05,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,8,461.9543151855469,461.9543151855469,461.9543151855469,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:05,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,16,922.994384765625,922.994384765625,922.994384765625,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:05,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,2,39.558860778808594,39.52657699584961,39.591148376464844,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:36,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,4,79.9734115600586,79.9734115600586,79.9734115600586,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:36,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,8,160.071044921875,160.071044921875,160.071044921875,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:36,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,16,321.4681091308594,321.4681091308594,321.4681091308594,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:24:36,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,2,116.56009674072266,116.56009674072266,116.56009674072266,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:17,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,4,232.43980407714844,232.43980407714844,232.43980407714844,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:17,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,8,464.5750732421875,464.5750732421875,464.5750732421875,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:17,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,16,926.3385009765625,926.3385009765625,926.3385009765625,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:17,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,2,120.68428802490234,120.68428802490234,120.68428802490234,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:58,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,4,241.15061950683594,241.15061950683594,241.15061950683594,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:58,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,8,492.5342102050781,492.5342102050781,492.5342102050781,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:58,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,16,1000.8460693359375,1000.8460693359375,1000.8460693359375,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:25:58,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,2,14556.626953125,14556.626953125,14556.626953125,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:26:42,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,4,14748.689453125,14748.689453125,14748.689453125,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:26:42,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,8,15132.814453125,15132.814453125,15132.814453125,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:26:42,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,16,15901.064453125,15901.064453125,15901.064453125,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:26:42,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,2,12488.501953125,12488.501953125,12488.501953125,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:27:10,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,4,19630.564453125,19630.564453125,19630.564453125,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:27:10,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,8,33914.6875,33914.6875,33914.6875,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:27:10,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,16,62482.9375,62482.9375,62482.9375,"{""T"": 4096, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 21:27:10,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,2,31.02783966064453,31.027551651000977,31.164947509765625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:30,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,4,60.88966369628906,60.88966369628906,60.88966369628906,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:30,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,8,121.08070373535156,121.08070373535156,121.08070373535156,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:30,0.4.0 +fused_linear_orpo_loss,liger,forward,speed,ms,B,B,16,244.36968994140625,244.36968994140625,244.36968994140625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:30,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,2,12.9093599319458,12.874624252319336,12.947936058044434,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:57,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,4,25.557632446289062,25.526700973510742,25.703763961791992,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:57,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,8,51.75590515136719,51.75590515136719,51.75590515136719,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:57,0.4.0 +fused_linear_orpo_loss,huggingface,forward,speed,ms,B,B,16,103.8515853881836,103.8515853881836,103.8515853881836,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:06:57,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,2,32.52537536621094,32.49258041381836,32.558170318603516,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:28,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,4,63.16300964355469,63.16300964355469,63.16300964355469,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:28,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,8,123.02518463134766,123.02518463134766,123.02518463134766,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:28,0.4.0 +fused_linear_orpo_loss,liger,full,speed,ms,B,B,16,247.44105529785156,247.44105529785156,247.44105529785156,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:28,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,2,39.32752227783203,39.32701873779297,39.32802200317383,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:59,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,4,77.9202880859375,77.9202880859375,77.9202880859375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:59,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,8,151.6084442138672,151.6084442138672,151.6084442138672,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:59,0.4.0 +fused_linear_orpo_loss,huggingface,full,speed,ms,B,B,16,304.4580993652344,304.4580993652344,304.4580993652344,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:07:59,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,2,8161.34619140625,8161.34619140625,8161.34619140625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:30,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,4,8209.361328125,8209.361328125,8209.361328125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:30,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,8,8305.392578125,8305.392578125,8305.392578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:30,0.4.0 +fused_linear_orpo_loss,liger,full,memory,MB,B,B,16,8497.455078125,8497.455078125,8497.455078125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:30,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,2,8645.314453125,8645.314453125,8645.314453125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:56,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,4,12184.330078125,12184.330078125,12184.330078125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:56,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,8,19262.361328125,19262.361328125,19262.361328125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:56,0.4.0 +fused_linear_orpo_loss,huggingface,full,memory,MB,B,B,16,33418.421875,33418.421875,33418.421875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-13 22:08:56,0.4.0 +fused_linear_cpo_loss,liger,forward,speed,ms,B,B,2,31.536447525024414,31.457439422607422,31.543052673339844,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:54:47,0.4.1 +fused_linear_cpo_loss,liger,forward,speed,ms,B,B,4,62.407745361328125,62.407745361328125,62.407745361328125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:54:47,0.4.1 +fused_linear_cpo_loss,liger,forward,speed,ms,B,B,8,123.64259338378906,123.64259338378906,123.64259338378906,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:54:47,0.4.1 +fused_linear_cpo_loss,liger,forward,speed,ms,B,B,16,245.66575622558594,245.66575622558594,245.66575622558594,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:54:47,0.4.1 +fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,2,14.516239166259766,14.514080047607422,14.52575969696045,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:20,0.4.1 +fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,4,26.087743759155273,25.943340301513672,26.269376754760742,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:20,0.4.1 +fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,8,51.85932922363281,51.85932922363281,51.85932922363281,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:20,0.4.1 +fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,16,104.99673461914062,104.99673461914062,104.99673461914062,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:20,0.4.1 +fused_linear_cpo_loss,liger,full,speed,ms,B,B,2,33.309967041015625,33.21604919433594,33.40388488769531,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:55,0.4.1 +fused_linear_cpo_loss,liger,full,speed,ms,B,B,4,63.053470611572266,63.053470611572266,63.053470611572266,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:55,0.4.1 +fused_linear_cpo_loss,liger,full,speed,ms,B,B,8,125.53849792480469,125.53849792480469,125.53849792480469,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:55,0.4.1 +fused_linear_cpo_loss,liger,full,speed,ms,B,B,16,250.22178649902344,250.22178649902344,250.22178649902344,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:55:55,0.4.1 +fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,2,39.45849609375,39.33102798461914,39.58596420288086,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:56:30,0.4.1 +fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,4,77.00272369384766,77.00272369384766,77.00272369384766,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:56:30,0.4.1 +fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,8,154.28419494628906,154.28419494628906,154.28419494628906,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:56:30,0.4.1 +fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,16,309.23162841796875,309.23162841796875,309.23162841796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:56:30,0.4.1 +fused_linear_cpo_loss,liger,full,memory,MB,B,B,2,8161.34619140625,8161.34619140625,8161.34619140625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:06,0.4.1 +fused_linear_cpo_loss,liger,full,memory,MB,B,B,4,8209.361328125,8209.361328125,8209.361328125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:06,0.4.1 +fused_linear_cpo_loss,liger,full,memory,MB,B,B,8,8305.392578125,8305.392578125,8305.392578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:06,0.4.1 +fused_linear_cpo_loss,liger,full,memory,MB,B,B,16,8497.455078125,8497.455078125,8497.455078125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:06,0.4.1 +fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,2,8645.314453125,8645.314453125,8645.314453125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:37,0.4.1 +fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,4,12184.330078125,12184.330078125,12184.330078125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:37,0.4.1 +fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,8,19262.361328125,19262.361328125,19262.361328125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:37,0.4.1 +fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,16,33418.42578125,33418.42578125,33418.42578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-14 16:57:37,0.4.1 +fused_linear_simpo_loss,liger,forward,speed,ms,B,B,2,30.28438377380371,30.107013702392578,30.284786224365234,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:26,0.4.1 +fused_linear_simpo_loss,liger,forward,speed,ms,B,B,4,58.80876922607422,58.80876922607422,58.80876922607422,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:26,0.4.1 +fused_linear_simpo_loss,liger,forward,speed,ms,B,B,8,117.96163177490234,117.96163177490234,117.96163177490234,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:26,0.4.1 +fused_linear_simpo_loss,liger,forward,speed,ms,B,B,16,235.60794067382812,235.60794067382812,235.60794067382812,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:26,0.4.1 +fused_linear_simpo_loss,huggingface,forward,speed,ms,B,B,2,14.513839721679688,14.510687828063965,14.517855644226074,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:56,0.4.1 +fused_linear_simpo_loss,huggingface,forward,speed,ms,B,B,4,28.78099250793457,28.72719383239746,28.792186737060547,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:56,0.4.1 +fused_linear_simpo_loss,huggingface,forward,speed,ms,B,B,8,52.5733757019043,52.5733757019043,52.5733757019043,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:56,0.4.1 +fused_linear_simpo_loss,huggingface,forward,speed,ms,B,B,16,104.44764709472656,104.44764709472656,104.44764709472656,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:27:56,0.4.1 +fused_linear_simpo_loss,liger,full,speed,ms,B,B,2,31.566062927246094,31.457612991333008,31.674514770507812,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:28:27,0.4.1 +fused_linear_simpo_loss,liger,full,speed,ms,B,B,4,61.4403190612793,61.4403190612793,61.4403190612793,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:28:27,0.4.1 +fused_linear_simpo_loss,liger,full,speed,ms,B,B,8,119.97705841064453,119.97705841064453,119.97705841064453,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:28:27,0.4.1 +fused_linear_simpo_loss,liger,full,speed,ms,B,B,16,238.13417053222656,238.13417053222656,238.13417053222656,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:28:27,0.4.1 +fused_linear_simpo_loss,huggingface,full,speed,ms,B,B,2,39.811119079589844,39.65474319458008,39.96749496459961,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:00,0.4.1 +fused_linear_simpo_loss,huggingface,full,speed,ms,B,B,4,77.20928192138672,77.20928192138672,77.20928192138672,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:00,0.4.1 +fused_linear_simpo_loss,huggingface,full,speed,ms,B,B,8,153.6952667236328,153.6952667236328,153.6952667236328,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:00,0.4.1 +fused_linear_simpo_loss,huggingface,full,speed,ms,B,B,16,307.7382507324219,307.7382507324219,307.7382507324219,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:00,0.4.1 +fused_linear_simpo_loss,liger,full,memory,MB,B,B,2,7675.3291015625,7675.3291015625,7675.3291015625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:33,0.4.1 +fused_linear_simpo_loss,liger,full,memory,MB,B,B,4,7723.3447265625,7723.3447265625,7723.3447265625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:33,0.4.1 +fused_linear_simpo_loss,liger,full,memory,MB,B,B,8,7819.3759765625,7819.3759765625,7819.3759765625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:33,0.4.1 +fused_linear_simpo_loss,liger,full,memory,MB,B,B,16,8011.4384765625,8011.4384765625,8011.4384765625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:29:33,0.4.1 +fused_linear_simpo_loss,huggingface,full,memory,MB,B,B,2,8645.314453125,8645.314453125,8645.314453125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:30:01,0.4.1 +fused_linear_simpo_loss,huggingface,full,memory,MB,B,B,4,12184.330078125,12184.330078125,12184.330078125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:30:01,0.4.1 +fused_linear_simpo_loss,huggingface,full,memory,MB,B,B,8,19262.361328125,19262.361328125,19262.361328125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:30:01,0.4.1 +fused_linear_simpo_loss,huggingface,full,memory,MB,B,B,16,33418.42578125,33418.42578125,33418.42578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2024-11-15 14:30:01,0.4.1 +distill_jsd_loss,liger,forward,speed,ms,BT,B x T,1024,7.735536098480225,7.729177474975586,7.798131465911865,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:58:46,0.4.2 +distill_jsd_loss,liger,forward,speed,ms,BT,B x T,2048,15.20411205291748,15.165056228637695,15.226079940795898,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:58:46,0.4.2 +distill_jsd_loss,liger,forward,speed,ms,BT,B x T,4096,30.159456253051758,30.126911163330078,30.165311813354492,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:58:46,0.4.2 +distill_jsd_loss,liger,forward,speed,ms,BT,B x T,8192,60.24163055419922,60.24163055419922,60.24163055419922,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:58:46,0.4.2 +distill_jsd_loss,torch,forward,speed,ms,BT,B x T,1024,10.906111717224121,10.903244972229004,10.91296672821045,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:18,0.4.2 +distill_jsd_loss,torch,forward,speed,ms,BT,B x T,2048,21.480207443237305,21.465139389038086,21.489286422729492,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:18,0.4.2 +distill_jsd_loss,torch,forward,speed,ms,BT,B x T,4096,42.96339416503906,42.96237564086914,42.96440887451172,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:18,0.4.2 +distill_jsd_loss,torch,forward,speed,ms,BT,B x T,8192,85.3946533203125,85.3946533203125,85.3946533203125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:18,0.4.2 +distill_jsd_loss,liger,full,speed,ms,BT,B x T,1024,8.312895774841309,8.310400009155273,8.326751708984375,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:51,0.4.2 +distill_jsd_loss,liger,full,speed,ms,BT,B x T,2048,15.770208358764648,15.767775535583496,15.774784088134766,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:51,0.4.2 +distill_jsd_loss,liger,full,speed,ms,BT,B x T,4096,30.922752380371094,30.920312881469727,30.927898406982422,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:51,0.4.2 +distill_jsd_loss,liger,full,speed,ms,BT,B x T,8192,60.70627212524414,60.70627212524414,60.70627212524414,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 07:59:51,0.4.2 +distill_jsd_loss,torch,full,speed,ms,BT,B x T,1024,28.72480010986328,28.718809127807617,28.728179931640625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:25,0.4.2 +distill_jsd_loss,torch,full,speed,ms,BT,B x T,2048,54.281761169433594,54.281761169433594,54.281761169433594,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:25,0.4.2 +distill_jsd_loss,torch,full,speed,ms,BT,B x T,4096,107.08905792236328,107.08905792236328,107.08905792236328,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:25,0.4.2 +distill_jsd_loss,torch,full,speed,ms,BT,B x T,8192,213.1598663330078,213.1598663330078,213.1598663330078,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:25,0.4.2 +distill_jsd_loss,liger,full,memory,MB,BT,B x T,1024,10913.541015625,10913.541015625,10913.541015625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:58,0.4.2 +distill_jsd_loss,liger,full,memory,MB,BT,B x T,2048,10941.548828125,10941.548828125,10941.548828125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:58,0.4.2 +distill_jsd_loss,liger,full,memory,MB,BT,B x T,4096,10997.564453125,10997.564453125,10997.564453125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:58,0.4.2 +distill_jsd_loss,liger,full,memory,MB,BT,B x T,8192,11109.595703125,11109.595703125,11109.595703125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:00:58,0.4.2 +distill_jsd_loss,torch,full,memory,MB,BT,B x T,1024,16174.0390625,16174.0390625,16174.0390625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:01:32,0.4.2 +distill_jsd_loss,torch,full,memory,MB,BT,B x T,2048,23713.05078125,23713.05078125,23713.05078125,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:01:32,0.4.2 +distill_jsd_loss,torch,full,memory,MB,BT,B x T,4096,38791.07421875,38791.07421875,38791.07421875,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:01:32,0.4.2 +distill_jsd_loss,torch,full,memory,MB,BT,B x T,8192,68947.1015625,68947.1015625,68947.1015625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA H100 80GB HBM3,2024-12-03 08:01:32,0.4.2 +kto_loss,liger,forward,speed,ms,B,Batch Size (B),2,3.9951679706573486,3.991487979888916,4.002252578735352,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:22:44,0.5.4 +kto_loss,liger,forward,speed,ms,B,Batch Size (B),4,7.8037919998168945,7.788575649261475,7.808595180511475,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:22:44,0.5.4 +kto_loss,liger,forward,speed,ms,B,Batch Size (B),8,15.43172836303711,15.430015563964844,15.4335355758667,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:22:44,0.5.4 +kto_loss,liger,forward,speed,ms,B,Batch Size (B),16,30.66864013671875,30.66431999206543,30.670501708984375,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:22:44,0.5.4 +kto_loss,liger,forward,speed,ms,B,Batch Size (B),32,61.1163215637207,61.1163215637207,61.1163215637207,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:22:44,0.5.4 +kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),2,3.8766400814056396,3.8680384159088135,3.8897151947021484,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:01,0.5.4 +kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),4,7.213727951049805,7.206470489501953,7.229574680328369,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:01,0.5.4 +kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),8,13.828800201416016,13.810944557189941,13.834943771362305,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:01,0.5.4 +kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),16,27.0930233001709,27.08517074584961,27.09713363647461,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:01,0.5.4 +kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),32,54.13715362548828,54.13715362548828,54.13715362548828,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:01,0.5.4 +kto_loss,liger,full,speed,ms,B,Batch Size (B),2,4.782928466796875,4.677459239959717,5.3430914878845215,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:18,0.5.4 +kto_loss,liger,full,speed,ms,B,Batch Size (B),4,8.517248153686523,8.481344223022461,8.561504364013672,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:18,0.5.4 +kto_loss,liger,full,speed,ms,B,Batch Size (B),8,16.547504425048828,16.513471603393555,16.678144454956055,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:18,0.5.4 +kto_loss,liger,full,speed,ms,B,Batch Size (B),16,31.891263961791992,31.819705963134766,32.274131774902344,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:18,0.5.4 +kto_loss,liger,full,speed,ms,B,Batch Size (B),32,62.953758239746094,62.953758239746094,62.953758239746094,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:18,0.5.4 +kto_loss,huggingface,full,speed,ms,B,Batch Size (B),2,6.201632022857666,6.163315296173096,6.314668655395508,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:35,0.5.4 +kto_loss,huggingface,full,speed,ms,B,Batch Size (B),4,11.156224250793457,11.142304420471191,11.207296371459961,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:35,0.5.4 +kto_loss,huggingface,full,speed,ms,B,Batch Size (B),8,21.249855041503906,21.231891632080078,21.264543533325195,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:35,0.5.4 +kto_loss,huggingface,full,speed,ms,B,Batch Size (B),16,41.55686569213867,41.536956787109375,41.57677459716797,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:35,0.5.4 +kto_loss,huggingface,full,speed,ms,B,Batch Size (B),32,81.56924438476562,81.56924438476562,81.56924438476562,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:35,0.5.4 +kto_loss,liger,full,memory,MB,B,Batch Size (B),2,2585.73876953125,2585.73876953125,2585.73876953125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:55,0.5.4 +kto_loss,liger,full,memory,MB,B,Batch Size (B),4,3348.9892578125,3348.9892578125,3348.9892578125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:55,0.5.4 +kto_loss,liger,full,memory,MB,B,Batch Size (B),8,3361.0048828125,3361.0048828125,3361.0048828125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:55,0.5.4 +kto_loss,liger,full,memory,MB,B,Batch Size (B),16,3385.0361328125,3385.0361328125,3385.0361328125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:55,0.5.4 +kto_loss,liger,full,memory,MB,B,Batch Size (B),32,3433.0986328125,3433.0986328125,3433.0986328125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:23:55,0.5.4 +kto_loss,huggingface,full,memory,MB,B,Batch Size (B),2,4341.74951171875,4341.74951171875,4341.74951171875,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:24:11,0.5.4 +kto_loss,huggingface,full,memory,MB,B,Batch Size (B),4,6099.26513671875,6099.26513671875,6099.26513671875,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:24:11,0.5.4 +kto_loss,huggingface,full,memory,MB,B,Batch Size (B),8,9613.298828125,9613.298828125,9613.298828125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:24:11,0.5.4 +kto_loss,huggingface,full,memory,MB,B,Batch Size (B),16,16643.365234375,16643.365234375,16643.365234375,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:24:11,0.5.4 +kto_loss,huggingface,full,memory,MB,B,Batch Size (B),32,30703.498046875,30703.498046875,30703.498046875,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H100 80GB HBM3,2025-03-03 08:24:11,0.5.4 +sparsemax,liger,forward,speed,ms,V,feature size,1024,0.41471999883651733,0.4126720130443573,0.42393600940704346,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:08,0.5.8 +sparsemax,liger,forward,speed,ms,V,feature size,2048,0.7608320116996765,0.7598080039024353,0.7628800272941589,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:08,0.5.8 +sparsemax,liger,forward,speed,ms,V,feature size,4096,1.4561280012130737,1.4540799856185913,1.4581760168075562,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:08,0.5.8 +sparsemax,liger,forward,speed,ms,V,feature size,8192,5.288959980010986,5.2848639488220215,5.29986572265625,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:08,0.5.8 +sparsemax,liger,forward,speed,ms,V,feature size,16384,10.734624862670898,10.729472160339355,11.096882820129395,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:08,0.5.8 +sparsemax,liger,forward,speed,ms,V,feature size,32768,21.729312896728516,21.7128963470459,22.20728302001953,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:08,0.5.8 +sparsemax,torch,forward,speed,ms,V,feature size,1024,0.42291200160980225,0.42188799381256104,0.42393600940704346,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:12,0.5.8 +sparsemax,torch,forward,speed,ms,V,feature size,2048,0.7782400250434875,0.7772160172462463,0.779263973236084,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:12,0.5.8 +sparsemax,torch,forward,speed,ms,V,feature size,4096,1.4940160512924194,1.491968035697937,1.4960639476776123,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:12,0.5.8 +sparsemax,torch,forward,speed,ms,V,feature size,8192,5.359615802764893,5.356544017791748,5.366579055786133,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:12,0.5.8 +sparsemax,torch,forward,speed,ms,V,feature size,16384,10.883584022521973,10.874879837036133,11.224268913269043,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:12,0.5.8 +sparsemax,torch,forward,speed,ms,V,feature size,32768,22.19878387451172,22.018457412719727,22.48888397216797,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:12,0.5.8 +sparsemax,liger,full,speed,ms,V,feature size,1024,0.4558719992637634,0.45558398962020874,0.45772799849510193,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:16,0.5.8 +sparsemax,liger,full,speed,ms,V,feature size,2048,0.8488960266113281,0.8478720188140869,0.8509439826011658,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:16,0.5.8 +sparsemax,liger,full,speed,ms,V,feature size,4096,1.6476160287857056,1.6465920209884644,1.6499264240264893,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:16,0.5.8 +sparsemax,liger,full,speed,ms,V,feature size,8192,5.664768218994141,5.660672187805176,5.681356906890869,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:16,0.5.8 +sparsemax,liger,full,speed,ms,V,feature size,16384,11.486207962036133,11.478015899658203,11.874713897705078,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:16,0.5.8 +sparsemax,liger,full,speed,ms,V,feature size,32768,23.457279205322266,23.289682388305664,23.76642608642578,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:16,0.5.8 +sparsemax,torch,full,speed,ms,V,feature size,1024,0.6021119952201843,0.6010879874229431,0.6041600108146667,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:20,0.5.8 +sparsemax,torch,full,speed,ms,V,feature size,2048,1.1212799549102783,1.119264006614685,1.1223039627075195,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:20,0.5.8 +sparsemax,torch,full,speed,ms,V,feature size,4096,2.1637120246887207,2.1616640090942383,2.165760040283203,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:20,0.5.8 +sparsemax,torch,full,speed,ms,V,feature size,8192,6.693888187408447,6.68723201751709,6.705561637878418,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:20,0.5.8 +sparsemax,torch,full,speed,ms,V,feature size,16384,13.523456573486328,13.518848419189453,13.878681182861328,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:20,0.5.8 +sparsemax,torch,full,speed,ms,V,feature size,32768,27.604991912841797,27.295129776000977,27.77518081665039,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:20,0.5.8 +sparsemax,liger,backward,speed,ms,V,feature size,1024,0.04403200000524521,0.043007999658584595,0.05222399905323982,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:22,0.5.8 +sparsemax,liger,backward,speed,ms,V,feature size,2048,0.08806400001049042,0.08713600039482117,0.08806400001049042,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:22,0.5.8 +sparsemax,liger,backward,speed,ms,V,feature size,4096,0.1884160041809082,0.1884160041809082,0.18943999707698822,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:22,0.5.8 +sparsemax,liger,backward,speed,ms,V,feature size,8192,0.374783992767334,0.37376001477241516,0.37486720085144043,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:22,0.5.8 +sparsemax,liger,backward,speed,ms,V,feature size,16384,0.7516160011291504,0.7505919933319092,0.7516160011291504,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:22,0.5.8 +sparsemax,liger,backward,speed,ms,V,feature size,32768,1.5738879442214966,1.572864055633545,1.575935959815979,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:22,0.5.8 +sparsemax,torch,backward,speed,ms,V,feature size,1024,0.1812479943037033,0.1802240014076233,0.18227200210094452,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,torch,backward,speed,ms,V,feature size,2048,0.34406399726867676,0.34406399726867676,0.34508800506591797,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,torch,backward,speed,ms,V,feature size,4096,0.6717439889907837,0.6707199811935425,0.6727679967880249,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,torch,backward,speed,ms,V,feature size,8192,1.3250559568405151,1.3241215944290161,1.3260799646377563,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,torch,backward,speed,ms,V,feature size,16384,2.629631996154785,2.628607988357544,2.6306560039520264,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,torch,backward,speed,ms,V,feature size,32768,5.236735820770264,5.235712051391602,5.239808082580566,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,liger,full,memory,MB,V,feature size,1024,82.03515625,82.03515625,82.03515625,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,liger,full,memory,MB,V,feature size,2048,164.0390625,164.0390625,164.0390625,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,liger,full,memory,MB,V,feature size,4096,328.046875,328.046875,328.046875,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,liger,full,memory,MB,V,feature size,8192,704.00048828125,704.00048828125,704.00048828125,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,liger,full,memory,MB,V,feature size,16384,1408.00048828125,1408.00048828125,1408.00048828125,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,liger,full,memory,MB,V,feature size,32768,2816.00048828125,2816.00048828125,2816.00048828125,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:25,0.5.8 +sparsemax,torch,full,memory,MB,V,feature size,1024,82.03515625,82.03515625,82.03515625,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:26,0.5.8 +sparsemax,torch,full,memory,MB,V,feature size,2048,164.0390625,164.0390625,164.0390625,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:26,0.5.8 +sparsemax,torch,full,memory,MB,V,feature size,4096,328.046875,328.046875,328.046875,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:26,0.5.8 +sparsemax,torch,full,memory,MB,V,feature size,8192,704.00048828125,704.00048828125,704.00048828125,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:26,0.5.8 +sparsemax,torch,full,memory,MB,V,feature size,16384,1408.00048828125,1408.00048828125,1408.00048828125,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:26,0.5.8 +sparsemax,torch,full,memory,MB,V,feature size,32768,2816.00048828125,2816.00048828125,2816.00048828125,"{""B"": 4, ""T"": 512, ""dim"": -1, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-28 00:38:26,0.5.8 +multi_token_attention,liger,forward,speed,ms,L,sequence length,32,0.01740800030529499,0.01740800030529499,0.018432000651955605,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:10,0.1.1 +multi_token_attention,liger,forward,speed,ms,L,sequence length,64,0.018432000651955605,0.01740800030529499,0.01945599913597107,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:10,0.1.1 +multi_token_attention,liger,forward,speed,ms,L,sequence length,128,0.023552000522613525,0.02252800017595291,0.02364799939095974,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:10,0.1.1 +multi_token_attention,liger,forward,speed,ms,L,sequence length,256,0.043007999658584595,0.04198399931192398,0.043007999658584595,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:10,0.1.1 +multi_token_attention,liger,forward,speed,ms,L,sequence length,512,0.12595200538635254,0.12492799758911133,0.12595200538635254,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:10,0.1.1 +multi_token_attention,liger,forward,speed,ms,L,sequence length,1024,0.5283839702606201,0.5253120064735413,0.5294079780578613,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:10,0.1.1 +multi_token_attention,torch,forward,speed,ms,L,sequence length,32,0.2467840015888214,0.24063999950885773,0.2529279887676239,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:11,0.1.1 +multi_token_attention,torch,forward,speed,ms,L,sequence length,64,0.24166400730609894,0.23756800591945648,0.24883200228214264,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:11,0.1.1 +multi_token_attention,torch,forward,speed,ms,L,sequence length,128,0.24268800020217896,0.2385600060224533,0.24985599517822266,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:11,0.1.1 +multi_token_attention,torch,forward,speed,ms,L,sequence length,256,0.24166400730609894,0.23873919248580933,0.24782079458236694,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:11,0.1.1 +multi_token_attention,torch,forward,speed,ms,L,sequence length,512,0.31334400177001953,0.3102720081806183,0.3213888108730316,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:11,0.1.1 +multi_token_attention,torch,forward,speed,ms,L,sequence length,1024,0.719871997833252,0.7167999744415283,0.7260159850120544,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:11,0.1.1 +multi_token_attention,liger,full,speed,ms,L,sequence length,32,0.9349120259284973,0.6543359756469727,0.9494400024414062,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:12,0.1.1 +multi_token_attention,liger,full,speed,ms,L,sequence length,64,0.6215680241584778,0.5631999969482422,0.8916991949081421,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:12,0.1.1 +multi_token_attention,liger,full,speed,ms,L,sequence length,128,0.5406720042228699,0.5335040092468262,0.550003170967102,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:12,0.1.1 +multi_token_attention,liger,full,speed,ms,L,sequence length,256,0.5631999969482422,0.5560320019721985,0.5674688220024109,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:12,0.1.1 +multi_token_attention,liger,full,speed,ms,L,sequence length,512,0.6430720090866089,0.6420480012893677,0.6430720090866089,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:12,0.1.1 +multi_token_attention,liger,full,speed,ms,L,sequence length,1024,2.4780800342559814,2.4770560264587402,2.479987144470215,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:12,0.1.1 +multi_token_attention,torch,full,speed,ms,L,sequence length,32,0.795199990272522,0.78438401222229,0.8038399815559387,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:13,0.1.1 +multi_token_attention,torch,full,speed,ms,L,sequence length,64,0.7362560033798218,0.6504960060119629,0.7464960217475891,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:13,0.1.1 +multi_token_attention,torch,full,speed,ms,L,sequence length,128,0.7680000066757202,0.6437439918518066,0.8105729818344116,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:13,0.1.1 +multi_token_attention,torch,full,speed,ms,L,sequence length,256,0.7685279846191406,0.7586879730224609,0.783519983291626,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:13,0.1.1 +multi_token_attention,torch,full,speed,ms,L,sequence length,512,0.9676799774169922,0.9625599980354309,0.9751039743423462,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:13,0.1.1 +multi_token_attention,torch,full,speed,ms,L,sequence length,1024,2.772480010986328,2.7688961029052734,2.7842559814453125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:13,0.1.1 +multi_token_attention,liger,backward,speed,ms,L,sequence length,32,0.334879994392395,0.3222528100013733,0.6912000179290771,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:14,0.1.1 +multi_token_attention,liger,backward,speed,ms,L,sequence length,64,0.23756800591945648,0.228166401386261,0.2629631757736206,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:14,0.1.1 +multi_token_attention,liger,backward,speed,ms,L,sequence length,128,0.29785600304603577,0.2519040107727051,0.3081727921962738,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:14,0.1.1 +multi_token_attention,liger,backward,speed,ms,L,sequence length,256,0.2590720057487488,0.24391679465770721,0.30832639336586,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:14,0.1.1 +multi_token_attention,liger,backward,speed,ms,L,sequence length,512,0.5171200037002563,0.5169600248336792,0.5181440114974976,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:14,0.1.1 +multi_token_attention,liger,backward,speed,ms,L,sequence length,1024,1.9578880071640015,1.9568639993667603,1.9615744352340698,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:14,0.1.1 +multi_token_attention,torch,backward,speed,ms,L,sequence length,32,0.09830400347709656,0.08908800035715103,0.20353920757770538,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,backward,speed,ms,L,sequence length,64,0.06348799914121628,0.062463998794555664,0.06348799914121628,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,backward,speed,ms,L,sequence length,128,0.09011200070381165,0.08908800035715103,0.09011200070381165,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,backward,speed,ms,L,sequence length,256,0.16383999586105347,0.16383999586105347,0.16486400365829468,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,backward,speed,ms,L,sequence length,512,0.52019202709198,0.5191680192947388,0.52019202709198,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,backward,speed,ms,L,sequence length,1024,1.9763200283050537,1.9752960205078125,1.9763200283050537,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,liger,full,memory,MB,L,sequence length,32,0.97412109375,0.97412109375,0.97412109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,liger,full,memory,MB,L,sequence length,64,1.53662109375,1.53662109375,1.53662109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,liger,full,memory,MB,L,sequence length,128,3.69287109375,3.69287109375,3.69287109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,liger,full,memory,MB,L,sequence length,256,13.068359375,13.068359375,13.068359375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,liger,full,memory,MB,L,sequence length,512,48.974609375,48.974609375,48.974609375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,liger,full,memory,MB,L,sequence length,1024,192.974609375,192.974609375,192.974609375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,full,memory,MB,L,sequence length,32,0.9599609375,0.9599609375,0.9599609375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,full,memory,MB,L,sequence length,64,1.4814453125,1.4814453125,1.4814453125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,full,memory,MB,L,sequence length,128,3.4736328125,3.4736328125,3.4736328125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,full,memory,MB,L,sequence length,256,12.19287109375,12.19287109375,12.19287109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,full,memory,MB,L,sequence length,512,45.47412109375,45.47412109375,45.47412109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +multi_token_attention,torch,full,memory,MB,L,sequence length,1024,178.97412109375,178.97412109375,178.97412109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-28 04:46:15,0.1.1 +softmax,liger,forward,speed,ms,N,hidden size,128,0.0071680000983178616,0.0071680000983178616,0.007942399941384792,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:04,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,256,0.008448000065982342,0.008191999979317188,0.009216000325977802,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:04,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,512,0.013311999849975109,0.01228800043463707,0.013311999849975109,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:04,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,1024,0.021503999829292297,0.021503999829292297,0.02252800017595291,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:04,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,2048,0.04095999896526337,0.04095999896526337,0.04198399931192398,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:04,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,4096,0.0798719972372055,0.0798719972372055,0.08089599758386612,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:04,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,128,0.006144000217318535,0.006144000217318535,0.0071680000983178616,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:07,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,256,0.008191999979317188,0.008191999979317188,0.009216000325977802,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:07,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,512,0.01228800043463707,0.01228800043463707,0.013311999849975109,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:07,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,1024,0.02252800017595291,0.02252800017595291,0.023552000522613525,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:07,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,2048,0.057583998888731,0.05734400078654289,0.058368001133203506,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:07,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,4096,0.08323200047016144,0.08294399827718735,0.08396799862384796,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:07,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,128,0.053247999399900436,0.04505600035190582,0.06172160431742668,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:10,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,256,0.05939200147986412,0.04198399931192398,0.11169920116662979,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:10,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,512,0.11577600240707397,0.07720960676670074,0.16793599724769592,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:10,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,1024,0.12492799758911133,0.10273279249668121,0.2982015907764435,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:10,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,2048,0.1013759970664978,0.10035199671983719,0.12902399897575378,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:10,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,4096,0.19660800695419312,0.19660800695419312,0.19763199985027313,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:10,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,128,0.013311999849975109,0.013311999849975109,0.013504000380635262,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:13,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,256,0.019152000546455383,0.018432000651955605,0.01945599913597107,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:13,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,512,0.03891199827194214,0.03788800165057182,0.03891199827194214,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:13,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,1024,0.08396799862384796,0.08396799862384796,0.08499199897050858,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:13,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,2048,0.18329599499702454,0.18329599499702454,0.18432000279426575,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:13,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,4096,0.3307519853115082,0.32972800731658936,0.33169281482696533,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:13,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,128,0.006335999816656113,0.006144000217318535,0.0071680000983178616,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:16,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,256,0.0071680000983178616,0.006144000217318535,0.0071680000983178616,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:16,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,512,0.008191999979317188,0.008191999979317188,0.009216000325977802,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:16,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,1024,0.013311999849975109,0.01228800043463707,0.013311999849975109,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:16,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,2048,0.02252800017595291,0.02252800017595291,0.023552000522613525,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:16,0.5.8 +softmax,liger,forward,speed,ms,N,hidden size,4096,0.04095999896526337,0.04095999896526337,0.04198399931192398,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:16,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,128,0.006144000217318535,0.005119999870657921,0.006144000217318535,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:19,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,256,0.006207999773323536,0.006144000217318535,0.0071680000983178616,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:19,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,512,0.008383999578654766,0.008191999979317188,0.009216000325977802,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:19,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,1024,0.014336000196635723,0.014336000196635723,0.014336000196635723,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:19,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,2048,0.05939200147986412,0.058368001133203506,0.05939200147986412,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:19,0.5.8 +softmax,torch,forward,speed,ms,N,hidden size,4096,0.06758400052785873,0.06675200164318085,0.06758400052785873,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:19,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,128,0.11472000181674957,0.09744639694690704,0.20684799551963806,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:22,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,256,0.15787199139595032,0.10769280046224594,0.20897281169891357,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:22,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,512,0.14028799533843994,0.0832064226269722,0.2879999876022339,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:22,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,1024,0.2088959962129593,0.11446399986743927,0.2972480058670044,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:22,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,2048,0.1443839967250824,0.09318400174379349,0.28278398513793945,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:22,0.5.8 +softmax,liger,full,speed,ms,N,hidden size,4096,0.11673600226640701,0.10035199671983719,0.28074881434440613,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:22,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,128,0.011264000087976456,0.010239999741315842,0.011264000087976456,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,256,0.013311999849975109,0.013311999849975109,0.013632000423967838,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,512,0.01945599913597107,0.01945599913597107,0.01945599913597107,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,1024,0.04198399931192398,0.04198399931192398,0.04224000126123428,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,2048,0.12595200538635254,0.12595200538635254,0.12697599828243256,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,speed,ms,N,hidden size,4096,0.19763199985027313,0.19660800695419312,0.19809921085834503,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,128,0.00244140625,0.00244140625,0.00244140625,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,256,0.0048828125,0.0048828125,0.0048828125,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,512,0.009765625,0.009765625,0.009765625,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,1024,0.01953125,0.01953125,0.01953125,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,2048,0.0390625,0.0390625,0.0390625,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,4096,0.078125,0.078125,0.078125,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,128,0.0029296875,0.0029296875,0.0029296875,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,256,0.005859375,0.005859375,0.005859375,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,512,0.01171875,0.01171875,0.01171875,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,1024,0.0234375,0.0234375,0.0234375,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,2048,0.046875,0.046875,0.046875,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,4096,0.09375,0.09375,0.09375,"{""M"": 2048, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,128,0.00244140625,0.00244140625,0.00244140625,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,256,0.00244140625,0.00244140625,0.00244140625,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,512,0.0048828125,0.0048828125,0.0048828125,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,1024,0.009765625,0.009765625,0.009765625,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,2048,0.01953125,0.01953125,0.01953125,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,liger,full,memory,MB,N,hidden size,4096,0.0390625,0.0390625,0.0390625,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:25,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,128,0.0029296875,0.0029296875,0.0029296875,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:26,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,256,0.0029296875,0.0029296875,0.0029296875,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:26,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,512,0.005859375,0.005859375,0.005859375,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:26,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,1024,0.01171875,0.01171875,0.01171875,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:26,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,2048,0.0234375,0.0234375,0.0234375,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:26,0.5.8 +softmax,torch,full,memory,MB,N,hidden size,4096,0.046875,0.046875,0.046875,"{""M"": 2048, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 3090,2025-04-30 16:11:26,0.5.8 +sparse_multi_token_attention,liger,forward,speed,ms,L,sequence length,32,0.31436800956726074,0.30646398663520813,0.319487988948822,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:12,0.5.8 +sparse_multi_token_attention,liger,forward,speed,ms,L,sequence length,64,0.3779039978981018,0.3678207993507385,0.38410240411758423,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:12,0.5.8 +sparse_multi_token_attention,liger,forward,speed,ms,L,sequence length,128,0.35020801424980164,0.3428351879119873,0.35839998722076416,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:12,0.5.8 +sparse_multi_token_attention,liger,forward,speed,ms,L,sequence length,256,0.5294079780578613,0.5283839702606201,0.5304319858551025,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:12,0.5.8 +sparse_multi_token_attention,liger,forward,speed,ms,L,sequence length,512,1.7315839529037476,1.7304960489273071,1.815551996231079,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:12,0.5.8 +sparse_multi_token_attention,liger,forward,speed,ms,L,sequence length,1024,6.465375900268555,6.462463855743408,6.718054294586182,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:12,0.5.8 +sparse_multi_token_attention,torch,forward,speed,ms,L,sequence length,32,0.5888000130653381,0.5826560258865356,0.5960000157356262,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:13,0.5.8 +sparse_multi_token_attention,torch,forward,speed,ms,L,sequence length,64,0.6010879874229431,0.5947520136833191,0.608128011226654,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:13,0.5.8 +sparse_multi_token_attention,torch,forward,speed,ms,L,sequence length,128,0.5816320180892944,0.5745791792869568,0.5908480286598206,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:13,0.5.8 +sparse_multi_token_attention,torch,forward,speed,ms,L,sequence length,256,0.8591359853744507,0.8529919981956482,0.8627520203590393,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:13,0.5.8 +sparse_multi_token_attention,torch,forward,speed,ms,L,sequence length,512,1.931391954421997,1.925772786140442,1.935705542564392,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:13,0.5.8 +sparse_multi_token_attention,torch,forward,speed,ms,L,sequence length,1024,6.76915168762207,6.761676788330078,7.009791851043701,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:13,0.5.8 +sparse_multi_token_attention,liger,full,speed,ms,L,sequence length,32,2.111056089401245,2.0716030597686768,2.137094497680664,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:16,0.5.8 +sparse_multi_token_attention,liger,full,speed,ms,L,sequence length,64,2.174975872039795,2.1364736557006836,2.297856092453003,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:16,0.5.8 +sparse_multi_token_attention,liger,full,speed,ms,L,sequence length,128,2.0894718170166016,2.073791980743408,2.1352319717407227,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:16,0.5.8 +sparse_multi_token_attention,liger,full,speed,ms,L,sequence length,256,2.137216091156006,1.8400319814682007,2.194175958633423,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:16,0.5.8 +sparse_multi_token_attention,liger,full,speed,ms,L,sequence length,512,2.2814719676971436,2.1872639656066895,2.2833151817321777,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:16,0.5.8 +sparse_multi_token_attention,liger,full,speed,ms,L,sequence length,1024,8.308735847473145,8.299519538879395,8.551424026489258,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:16,0.5.8 +sparse_multi_token_attention,torch,full,speed,ms,L,sequence length,32,1.5749119520187378,1.498412847518921,2.170527935028076,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:17,0.5.8 +sparse_multi_token_attention,torch,full,speed,ms,L,sequence length,64,1.494047999382019,1.482604742050171,1.5207936763763428,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:17,0.5.8 +sparse_multi_token_attention,torch,full,speed,ms,L,sequence length,128,1.4581760168075562,1.4419968128204346,2.1133759021759033,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:17,0.5.8 +sparse_multi_token_attention,torch,full,speed,ms,L,sequence length,256,1.7448960542678833,1.7180671691894531,1.7537024021148682,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:17,0.5.8 +sparse_multi_token_attention,torch,full,speed,ms,L,sequence length,512,2.796544075012207,2.7762560844421387,2.8190720081329346,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:17,0.5.8 +sparse_multi_token_attention,torch,full,speed,ms,L,sequence length,1024,9.511823654174805,9.501286506652832,9.787391662597656,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:17,0.5.8 +sparse_multi_token_attention,liger,backward,speed,ms,L,sequence length,32,0.3544960021972656,0.33546239137649536,0.8041215538978577,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,liger,backward,speed,ms,L,sequence length,64,0.32897597551345825,0.32051199674606323,0.3438591957092285,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,liger,backward,speed,ms,L,sequence length,128,0.30931198596954346,0.3002240061759949,0.3197120130062103,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,liger,backward,speed,ms,L,sequence length,256,0.31334400177001953,0.2956160008907318,0.3251904249191284,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,liger,backward,speed,ms,L,sequence length,512,0.447488009929657,0.44646400213241577,0.4485119879245758,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,liger,backward,speed,ms,L,sequence length,1024,1.8585599660873413,1.8574656248092651,1.861631989479065,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,torch,backward,speed,ms,L,sequence length,32,0.25804799795150757,0.24883200228214264,0.30926719307899475,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,torch,backward,speed,ms,L,sequence length,64,0.25804799795150757,0.2514623999595642,0.26668161153793335,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,torch,backward,speed,ms,L,sequence length,128,0.24075199663639069,0.2303999960422516,0.25194239616394043,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,torch,backward,speed,ms,L,sequence length,256,0.24686399102210999,0.23756800591945648,0.2550272047519684,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,torch,backward,speed,ms,L,sequence length,512,0.7045120000839233,0.704479992389679,0.7063615918159485,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,torch,backward,speed,ms,L,sequence length,1024,2.698431968688965,2.697216033935547,2.7013120651245117,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:18,0.5.8 +sparse_multi_token_attention,liger,full,memory,MB,L,sequence length,32,0.3603515625,0.3603515625,0.3603515625,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,liger,full,memory,MB,L,sequence length,64,1.4189453125,1.4189453125,1.4189453125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,liger,full,memory,MB,L,sequence length,128,5.6455078125,5.6455078125,5.6455078125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,liger,full,memory,MB,L,sequence length,256,22.53662109375,22.53662109375,22.53662109375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,liger,full,memory,MB,L,sequence length,512,90.06884765625,90.06884765625,90.06884765625,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,liger,full,memory,MB,L,sequence length,1024,360.13330078125,360.13330078125,360.13330078125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,torch,full,memory,MB,L,sequence length,32,0.45263671875,0.45263671875,0.45263671875,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,torch,full,memory,MB,L,sequence length,64,1.7685546875,1.7685546875,1.7685546875,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,torch,full,memory,MB,L,sequence length,128,7.04833984375,7.04833984375,7.04833984375,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,torch,full,memory,MB,L,sequence length,256,28.15478515625,28.15478515625,28.15478515625,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,torch,full,memory,MB,L,sequence length,512,112.55517578125,112.55517578125,112.55517578125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +sparse_multi_token_attention,torch,full,memory,MB,L,sequence length,1024,450.10595703125,450.10595703125,450.10595703125,"{""B"": 2, ""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-04-30 17:22:19,0.5.8 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.236735999584198,0.16073599457740784,0.24985599517822266,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.22323200106620789,0.21503999829292297,0.2323904037475586,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.24268800020217896,0.2295808047056198,0.25088000297546387,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.3307519853115082,0.32805120944976807,0.3317759931087494,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,0.8540160059928894,0.851967990398407,0.8595455884933472,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,2.3658719062805176,2.3617537021636963,2.368511915206909,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,8.466431617736816,8.447999954223633,8.480768203735352,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:08:54,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,5.16915225982666,5.143871784210205,5.297952175140381,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,10.244048118591309,10.094131469726562,10.48145866394043,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,20.196895599365234,20.145601272583008,21.581132888793945,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,42.183536529541016,41.2415771484375,43.12549591064453,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,77.73798370361328,77.73798370361328,77.73798370361328,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,172.90853881835938,172.90853881835938,172.90853881835938,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,346.5686950683594,346.5686950683594,346.5686950683594,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:01,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,2.723423957824707,2.68287992477417,2.7842559814453125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,2.6542398929595947,2.6169726848602295,2.68984317779541,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,2.595871925354004,2.1286911964416504,2.6818559169769287,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,2.738736152648926,2.7115519046783447,2.8180480003356934,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,2.83457612991333,2.805759906768799,2.88972806930542,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,6.529168128967285,6.525951862335205,6.66664981842041,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,23.742895126342773,23.660747528076172,23.825515747070312,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:14,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,6.841343879699707,6.725196838378906,6.972832202911377,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,11.825152397155762,11.683839797973633,12.080537796020508,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,21.856351852416992,21.36012077331543,21.95940589904785,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,42.70033264160156,42.545169830322266,42.855499267578125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,87.9656982421875,87.9656982421875,87.9656982421875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,181.77536010742188,181.77536010742188,181.77536010742188,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,368.0634765625,368.0634765625,368.0634765625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:21,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.5920320153236389,0.5674688220024109,1.3856768608093262,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.6430720090866089,0.6318399906158447,0.6610943675041199,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.6456320285797119,0.6359040141105652,0.6676480174064636,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,0.7014399766921997,0.6911231875419617,0.7275007963180542,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,1.4684159755706787,1.4663679599761963,1.4704639911651611,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,4.150223731994629,4.14717435836792,4.234445095062256,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,15.17465591430664,14.853119850158691,15.310848236083984,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:22,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.6000639796257019,0.5832703709602356,1.2799999713897705,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.5550079941749573,0.5488640069961548,0.5914624333381653,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.5470079779624939,0.5406720042228699,0.562175989151001,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,0.8714240193367004,0.8617984056472778,1.2751424312591553,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,2.3746559619903564,2.3727169036865234,2.3797760009765625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,8.019968032836914,8.00870418548584,8.2227201461792,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,28.92291259765625,28.684505462646484,28.97941780090332,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:25,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.23756800591945648,0.22630399465560913,0.24985599517822266,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.25088000297546387,0.24187520146369934,0.25964802503585815,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.43110400438308716,0.42920318245887756,0.43212801218032837,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,1.0199040174484253,1.0147839784622192,1.0281280279159546,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,2.584575891494751,2.578432083129883,2.593791961669922,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,7.8611040115356445,7.851212978363037,8.14100456237793,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,27.072511672973633,27.043020248413086,27.129650115966797,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:32,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,5.303808212280273,5.205196857452393,5.414611339569092,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,10.352640151977539,10.268671989440918,10.546982765197754,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,20.696575164794922,20.600217819213867,22.168373107910156,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,40.9251823425293,39.459224700927734,42.39113998413086,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,84.20972442626953,84.20972442626953,84.20972442626953,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,165.5727996826172,165.5727996826172,165.5727996826172,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,365.4942626953125,365.4942626953125,365.4942626953125,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:38,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,2.5410561561584473,2.5221376419067383,2.574540853500366,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,2.6214399337768555,2.5966720581054688,2.66780161857605,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,2.6818559169769287,2.660710334777832,2.7396223545074463,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,2.9624319076538086,2.959359884262085,2.973695993423462,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,7.516160011291504,7.5141119956970215,7.782809734344482,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,22.99033546447754,22.859058380126953,23.101655960083008,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,79.14390563964844,79.14390563964844,79.14390563964844,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:09:52,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,6.206463813781738,6.177548885345459,6.346368312835693,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,11.45395278930664,11.369497299194336,11.57201862335205,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,21.295616149902344,20.8918514251709,22.428876876831055,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,46.485904693603516,44.799137115478516,48.172672271728516,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,87.60115051269531,87.60115051269531,87.60115051269531,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,210.36146545410156,210.36146545410156,210.36146545410156,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,456.848388671875,456.848388671875,456.848388671875,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:00,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.5756800174713135,0.45319682359695435,0.7064127922058105,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.5908480286598206,0.48742398619651794,0.6028479933738708,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.915615975856781,0.8775680065155029,0.9175040125846863,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,1.9450880289077759,1.9351999759674072,1.9651199579238892,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,4.930560111999512,4.915200233459473,5.046477317810059,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,15.102832794189453,14.952447891235352,15.31494426727295,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,52.104190826416016,52.104190826416016,52.104190826416016,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:02,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.4843519926071167,0.4761984050273895,0.6077119708061218,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.5319839715957642,0.5222399830818176,0.5335040092468262,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,1.1182080507278442,1.1151360273361206,1.120255947113037,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,2.5815041065216064,2.5763840675354004,2.5960447788238525,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,7.123968124389648,7.087513446807861,7.359897613525391,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,24.104448318481445,24.077312469482422,24.161880493164062,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,86.40716552734375,86.40716552734375,86.40716552734375,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:05,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.2467840015888214,0.17902079224586487,0.25702399015426636,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.23756800591945648,0.23654399812221527,0.24885760247707367,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.4567039906978607,0.45158401131629944,0.4638719856739044,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.8017920255661011,0.7946239709854126,0.8048639893531799,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,1.9527679681777954,1.9476544857025146,1.9595264196395874,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,5.405695915222168,5.392384052276611,5.651423931121826,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,18.608959197998047,18.311372756958008,18.646629333496094,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:12,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,6.554111957550049,6.130688190460205,6.872096061706543,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,13.195263862609863,13.134265899658203,13.464166641235352,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,24.001535415649414,23.594995498657227,25.934438705444336,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,50.334720611572266,50.334720611572266,50.334720611572266,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,107.2701416015625,107.2701416015625,107.2701416015625,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,218.13658142089844,218.13658142089844,218.13658142089844,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,457.2313537597656,457.2313537597656,457.2313537597656,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:20,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,2.623487949371338,2.605638265609741,2.6442177295684814,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,2.6389598846435547,2.6225087642669678,2.6781694889068604,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,2.613312005996704,2.589139223098755,2.6998207569122314,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,2.7299840450286865,2.7037951946258545,2.783027172088623,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,5.588992118835449,5.584896087646484,5.632409572601318,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,15.91859245300293,15.853568077087402,16.029695510864258,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,54.28019332885742,54.28019332885742,54.28019332885742,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:34,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,8.281087875366211,8.076288223266602,8.5731840133667,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,14.909952163696289,14.721952438354492,15.562975883483887,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,25.10848045349121,25.013248443603516,25.180980682373047,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,53.98118209838867,53.98118209838867,53.98118209838867,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,115.51538848876953,115.51538848876953,115.51538848876953,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,234.2144012451172,234.2144012451172,234.2144012451172,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,493.1143798828125,493.1143798828125,493.1143798828125,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:43,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.6873279809951782,0.6780927777290344,0.8112127780914307,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.6923519968986511,0.6756608486175537,0.8371520042419434,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.7854080200195312,0.7739391922950745,0.7946239709854126,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,1.5523840188980103,1.5431679487228394,1.5880192518234253,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,3.635200023651123,3.634176015853882,3.637446403503418,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,10.225664138793945,10.196991920471191,10.515456199645996,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,35.736061096191406,35.612876892089844,35.859249114990234,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.4935680031776428,0.4843519926071167,1.2861696481704712,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.5950400233268738,0.4885439872741699,0.7454720735549927,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.9082880020141602,0.8939520120620728,1.2302591800689697,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,1.994752049446106,1.9916800260543823,2.002943992614746,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,5.427199840545654,5.400953769683838,5.5943169593811035,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,16.917503356933594,16.85626792907715,17.202789306640625,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,58.775550842285156,58.775550842285156,58.775550842285156,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:48,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.16998399794101715,0.159743994474411,0.24968959391117096,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.15515199303627014,0.14643199741840363,0.16281600296497345,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.16998399794101715,0.159743994474411,0.25088000297546387,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.3307519853115082,0.32767999172210693,0.3317759931087494,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,0.8550400137901306,0.8529919981956482,0.8581119775772095,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,2.3664638996124268,2.36456298828125,2.371583938598633,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,8.253439903259277,8.21452808380127,8.534015655517578,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:52,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,5.056511878967285,4.674380779266357,5.254271984100342,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,10.41360092163086,10.147839546203613,10.88619613647461,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,21.108095169067383,19.98341178894043,22.000703811645508,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,39.93907165527344,39.49793243408203,40.380210876464844,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,87.47724914550781,87.47724914550781,87.47724914550781,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,162.8107147216797,162.8107147216797,162.8107147216797,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,318.89202880859375,318.89202880859375,318.89202880859375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:58,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,2.756608009338379,2.50598406791687,2.862694263458252,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,2.683903932571411,2.656268835067749,2.720358371734619,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,2.6729280948638916,2.649907112121582,2.703104019165039,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,2.8049919605255127,2.7712254524230957,2.848358392715454,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,2.8816640377044678,2.8426239490509033,2.966118335723877,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,6.523903846740723,6.52185583114624,6.534143924713135,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,23.48236846923828,23.36788558959961,23.587430953979492,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:10:59,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,6.210592269897461,6.149964809417725,6.439935684204102,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,11.412479400634766,11.000422477722168,12.122776985168457,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,21.02124786376953,20.722354888916016,21.280357360839844,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,44.49420928955078,43.21909713745117,45.769317626953125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,77.97862243652344,77.97862243652344,77.97862243652344,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,169.87033081054688,169.87033081054688,169.87033081054688,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,360.7623596191406,360.7623596191406,360.7623596191406,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:06,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.6484479904174805,0.5443072319030762,1.446675181388855,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.5460799932479858,0.536575973033905,0.6473984122276306,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.5612640380859375,0.5377407670021057,0.6634495854377747,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,0.6347839832305908,0.6327999830245972,0.7219520211219788,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,1.4684159755706787,1.4624768495559692,1.4744960069656372,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,4.150784015655518,4.148223876953125,4.164403438568115,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,15.233535766601562,14.96678352355957,15.318016052246094,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.596992015838623,0.5801728367805481,1.2581120729446411,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.5565760135650635,0.456928014755249,0.5724160075187683,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.5560640096664429,0.4616512060165405,0.5724160075187683,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,0.8714240193367004,0.8622080087661743,1.2775424718856812,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,2.3746559619903564,2.371583938598633,2.3776895999908447,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,8.032719612121582,8.015257835388184,8.314061164855957,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,29.113344192504883,28.672204971313477,29.20366096496582,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,32.525390625,32.525390625,32.525390625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,37.7734375,37.7734375,37.7734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,53.2734375,53.2734375,53.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,102.2734375,102.2734375,102.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,272.2734375,272.2734375,272.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,900.2734375,900.2734375,900.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,3308.2734375,3308.2734375,3308.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:10,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,32.53125,32.53125,32.53125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,36.8046875,36.8046875,36.8046875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,53.3359375,53.3359375,53.3359375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,110.5234375,110.5234375,110.5234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,321.2734375,321.2734375,321.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,1128.2734375,1128.2734375,1128.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,4284.2734375,4284.2734375,4284.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:17,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,55.2880859375,55.2880859375,55.2880859375,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,72.28515625,72.28515625,72.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,119.03515625,119.03515625,119.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,265.28515625,265.28515625,265.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,775.28515625,775.28515625,775.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,2659.28515625,2659.28515625,2659.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,9883.28515625,9883.28515625,9883.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:18,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,55.2919921875,55.2919921875,55.2919921875,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,70.05078125,70.05078125,70.05078125,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,118.34765625,118.34765625,118.34765625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,289.53515625,289.53515625,289.53515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,920.28515625,920.28515625,920.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,3335.28515625,3335.28515625,3335.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,12779.28515625,12779.28515625,12779.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:27,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,74.80078125,74.80078125,74.80078125,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,83.296875,83.296875,83.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,114.296875,114.296875,114.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,212.296875,212.296875,212.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,552.296875,552.296875,552.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,1808.296875,1808.296875,1808.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,6624.296875,6624.296875,6624.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:28,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,74.8046875,74.8046875,74.8046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,82.31640625,82.31640625,82.31640625,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,114.359375,114.359375,114.359375,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,228.546875,228.546875,228.546875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,649.296875,649.296875,649.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,2260.296875,2260.296875,2260.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,8560.296875,8560.296875,8560.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:38,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,32.525390625,32.525390625,32.525390625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,37.7734375,37.7734375,37.7734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,53.2734375,53.2734375,53.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,102.2734375,102.2734375,102.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,272.2734375,272.2734375,272.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,900.2734375,900.2734375,900.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,3308.2734375,3308.2734375,3308.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,32.53125,32.53125,32.53125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,36.8046875,36.8046875,36.8046875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,53.3359375,53.3359375,53.3359375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,110.5234375,110.5234375,110.5234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,321.2734375,321.2734375,321.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,1128.2734375,1128.2734375,1128.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,4284.2734375,4284.2734375,4284.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA GeForce RTX 3090,2025-05-27 15:11:46,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.25600001215934753,0.25436800718307495,0.2605184018611908,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.2569279968738556,0.25494399666786194,0.26105600595474243,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.25676798820495605,0.2550591826438904,0.2598848044872284,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.25841599702835083,0.25681281089782715,0.2625727951526642,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,0.3150399923324585,0.31407999992370605,0.31611520051956177,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,0.8260959982872009,0.8238016366958618,0.828614354133606,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,2.5686399936676025,2.557523012161255,2.5757951736450195,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:08,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,5.276463985443115,5.270419120788574,5.286643028259277,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,10.498432159423828,10.476134300231934,10.51439380645752,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,20.82036781311035,20.771360397338867,20.881420135498047,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,42.07323455810547,41.776065826416016,42.370399475097656,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,81.8509750366211,81.8509750366211,81.8509750366211,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,165.88720703125,165.88720703125,165.88720703125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,331.2662658691406,331.2662658691406,331.2662658691406,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:14,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,0.8993600010871887,0.8924031853675842,0.9097279906272888,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,0.8939200043678284,0.8890752196311951,0.9034687876701355,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,0.9244480133056641,0.9180480241775513,0.940447986125946,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,0.9229600429534912,0.915289580821991,0.9307839870452881,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,0.9950560331344604,0.9915199875831604,0.9971520304679871,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,2.5537919998168945,2.548985481262207,2.5564353466033936,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,7.698319911956787,7.67669153213501,7.713951587677002,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:25,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,5.840767860412598,5.819551944732666,5.864096164703369,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,11.064079284667969,11.050003051757812,11.102252960205078,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,21.443504333496094,21.364646911621094,21.61541748046875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,42.16088104248047,42.137290954589844,42.18446731567383,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,84.43017578125,84.43017578125,84.43017578125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,169.27821350097656,169.27821350097656,169.27821350097656,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,342.5223388671875,342.5223388671875,342.5223388671875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:31,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.49110400676727295,0.4891200065612793,0.49513599276542664,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.4911839962005615,0.4894847869873047,0.4949440062046051,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.5103520154953003,0.5084800124168396,0.5146496295928955,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,0.5199040174484253,0.5182399749755859,0.5254335999488831,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,0.6806079745292664,0.6792960166931152,0.681990385055542,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,1.7373919486999512,1.7352639436721802,1.7395071983337402,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,5.2151360511779785,5.205132484436035,5.221510410308838,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:32,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.4123840034008026,0.41091200709342957,0.4163135886192322,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.4136800169944763,0.41203200817108154,0.4168703854084015,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.4320639967918396,0.4301888048648834,0.4355071783065796,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,0.44307199120521545,0.44010239839553833,0.4480448067188263,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,0.9624000191688538,0.9609023928642273,0.9633920192718506,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,2.6429600715637207,2.641439914703369,2.644223928451538,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,8.974464416503906,8.973376274108887,8.97913646697998,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:35,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.2598559856414795,0.2580096125602722,0.2628991901874542,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.2602880001068115,0.25900799036026,0.26241281628608704,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.2643519937992096,0.2627519965171814,0.26796799898147583,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.41286399960517883,0.4122239947319031,0.4134399890899658,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,0.9781439900398254,0.9763264060020447,0.9801728129386902,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,2.659600019454956,2.655103921890259,2.6648640632629395,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,8.184944152832031,8.175705909729004,8.197542190551758,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:40,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,5.3048319816589355,5.287481784820557,5.315853118896484,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,10.493408203125,10.434623718261719,10.539365768432617,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,20.872079849243164,20.860185623168945,21.320632934570312,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,41.84241485595703,41.80018615722656,41.884647369384766,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,84.96883392333984,84.96883392333984,84.96883392333984,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,169.7915802001953,169.7915802001953,169.7915802001953,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,345.4809265136719,345.4809265136719,345.4809265136719,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:47,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,0.9144960045814514,0.9068800210952759,0.9251199960708618,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,0.9177280068397522,0.9107391834259033,0.9262208342552185,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,0.9360480308532715,0.9290496110916138,0.949785590171814,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,1.2921760082244873,1.289574384689331,1.2943040132522583,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,2.9243199825286865,2.919097423553467,2.9282751083374023,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,7.83568000793457,7.829171180725098,7.843168258666992,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,24.4779052734375,24.40936279296875,24.545881271362305,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:25:56,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,5.912464141845703,5.879615783691406,5.923999786376953,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,11.05232048034668,11.035250663757324,11.079456329345703,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,21.471296310424805,21.445714950561523,21.49998664855957,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,42.718048095703125,42.69863510131836,42.73746109008789,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,86.00204467773438,86.00204467773438,86.00204467773438,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,177.3928985595703,177.3928985595703,177.3928985595703,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,373.61773681640625,373.61773681640625,373.61773681640625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:03,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.5130239725112915,0.5107200145721436,0.5175104141235352,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.5187360048294067,0.5168319940567017,0.522816002368927,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.5284639596939087,0.5261759757995605,0.5319616198539734,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,0.8799999952316284,0.8791552186012268,0.8812223672866821,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,1.9606720209121704,1.9588288068771362,1.9625920057296753,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,5.239616394042969,5.233331203460693,5.246374607086182,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,16.295886993408203,16.174047470092773,16.315935134887695,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:05,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.4262079894542694,0.42505601048469543,0.42970240116119385,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.43747198581695557,0.43620482087135315,0.4399871826171875,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.5542719960212708,0.5531839728355408,0.555072009563446,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,1.0854079723358154,1.0841728448867798,1.0862784385681152,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,2.6914560794830322,2.6902334690093994,2.6927361488342285,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,8.072175979614258,8.052319526672363,8.081612586975098,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,27.25152015686035,27.248275756835938,27.25334358215332,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:07,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.26579201221466064,0.26371198892593384,0.2690303921699524,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.26337599754333496,0.26162558794021606,0.2659648060798645,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.264384001493454,0.2627967894077301,0.267276793718338,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.3535360097885132,0.3527039885520935,0.3543359935283661,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,0.7347840070724487,0.7331455945968628,0.7361727952957153,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,1.8545279502868652,1.850592017173767,1.8574399948120117,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,5.953392028808594,5.927840232849121,5.962080001831055,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:14,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,6.691328048706055,6.674118518829346,6.712192058563232,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,13.332127571105957,13.322579383850098,13.362988471984863,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,26.70470428466797,26.678035736083984,27.087322235107422,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,52.936126708984375,52.936126708984375,52.936126708984375,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,107.26537322998047,107.26537322998047,107.26537322998047,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,213.9727020263672,213.9727020263672,213.9727020263672,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,430.3240966796875,430.3240966796875,430.3240966796875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:22,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,0.912992000579834,0.8976320028305054,0.9327296018600464,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,0.9216639995574951,0.9107776284217834,0.9301823973655701,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,0.915615975856781,0.9078848361968994,0.9261952042579651,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,1.1379199028015137,1.1355520486831665,1.1407424211502075,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,2.277343988418579,2.268371343612671,2.2814719676971436,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,5.6143999099731445,5.608166217803955,5.673030376434326,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,17.534591674804688,17.516069412231445,17.57676124572754,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:32,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,7.29852819442749,7.287238597869873,7.318784236907959,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,13.901632308959961,13.893203735351562,13.942361831665039,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,27.261056900024414,27.254297256469727,27.288244247436523,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,54.26707077026367,54.26707077026367,54.26707077026367,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,108.40013122558594,108.40013122558594,108.40013122558594,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,220.19622802734375,220.19622802734375,220.19622802734375,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,453.9944763183594,453.9944763183594,453.9944763183594,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:40,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.49564799666404724,0.4941760003566742,0.49819520115852356,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.5055680274963379,0.5036479830741882,0.5097920298576355,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.5073280334472656,0.5049920082092285,0.5109120011329651,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,0.7868000268936157,0.7859584093093872,0.7878463864326477,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,1.5349119901657104,1.5336960554122925,1.5368640422821045,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,3.791167974472046,3.787168025970459,3.802060842514038,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,11.613519668579102,11.596006393432617,11.618464469909668,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:42,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.41388800740242004,0.412447988986969,0.417279988527298,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.42691200971603394,0.42473599314689636,0.4324415922164917,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.4886400103569031,0.48771199584007263,0.48993921279907227,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,0.9216960072517395,0.9203839898109436,0.9231168031692505,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,1.9877119064331055,1.9866175651550293,1.9888639450073242,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,5.659264087677002,5.653772830963135,5.6628031730651855,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,18.87718391418457,18.870214462280273,18.878368377685547,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:45,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,64,0.26070401072502136,0.258950412273407,0.26361599564552307,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,128,0.2584800124168396,0.256985604763031,0.26101118326187134,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,256,0.25942400097846985,0.25811201333999634,0.2618303894996643,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,512,0.26097601652145386,0.25948798656463623,0.2640959918498993,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,1024,0.3149600028991699,0.3140160143375397,0.31593599915504456,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,2048,0.8244799971580505,0.8216319680213928,0.8271167874336243,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,liger,forward,speed,ms,seq_len,sequence length,4096,2.5662078857421875,2.5587263107299805,2.5770816802978516,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:49,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,64,5.195775985717773,5.172947406768799,5.230342388153076,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,128,10.488927841186523,10.467231750488281,10.511955261230469,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,256,21.20012664794922,21.1026554107666,21.275672912597656,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,512,43.42755126953125,42.99705123901367,43.858055114746094,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,1024,84.55020904541016,84.55020904541016,84.55020904541016,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,2048,169.3335418701172,169.3335418701172,169.3335418701172,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,torch,forward,speed,ms,seq_len,sequence length,4096,340.14495849609375,340.14495849609375,340.14495849609375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:55,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,64,0.8945279717445374,0.886732816696167,0.9055423736572266,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,128,0.8908159732818604,0.8847360014915466,0.8983359932899475,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,256,0.9086400270462036,0.9012479782104492,0.9151040315628052,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,512,0.9225280284881592,0.9153919816017151,0.9314560294151306,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,1024,0.9986559748649597,0.9929599761962891,1.0019199848175049,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,2048,2.5703680515289307,2.56607985496521,2.574105739593506,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,liger,full,speed,ms,seq_len,sequence length,4096,7.78985595703125,7.7626495361328125,7.792575836181641,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:26:56,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,64,5.764095783233643,5.736550331115723,5.7790656089782715,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,128,11.027040481567383,11.009875297546387,11.10332202911377,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,256,21.499038696289062,21.467283248901367,21.521759033203125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,512,42.39520263671875,42.34148025512695,42.44892120361328,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,1024,85.2570571899414,85.2570571899414,85.2570571899414,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,2048,172.73379516601562,172.73379516601562,172.73379516601562,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,torch,full,speed,ms,seq_len,sequence length,4096,347.4947509765625,347.4947509765625,347.4947509765625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:03,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,64,0.4941760003566742,0.49265921115875244,0.4977791905403137,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,128,0.49348801374435425,0.49185919761657715,0.4974527955055237,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,256,0.5101760029792786,0.5087360143661499,0.5148288011550903,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,512,0.5200639963150024,0.5186240077018738,0.5237439870834351,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,1024,0.6887840032577515,0.6859776377677917,0.6903167963027954,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,2048,1.7373759746551514,1.7341376543045044,1.7395455837249756,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,liger,backward,speed,ms,seq_len,sequence length,4096,5.201104164123535,5.196633815765381,5.208876609802246,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:04,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,64,0.4107840061187744,0.40908798575401306,0.41468799114227295,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,128,0.4121600091457367,0.4106624126434326,0.4156480133533478,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,256,0.4296959936618805,0.42847999930381775,0.4339391887187958,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,512,0.43406400084495544,0.4329279959201813,0.43656960129737854,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,1024,0.9568639993667603,0.9556096196174622,0.9582463502883911,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,2048,2.6357598304748535,2.634399890899658,2.6394240856170654,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,torch,backward,speed,ms,seq_len,sequence length,4096,8.944831848144531,8.943455696105957,8.947711944580078,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:06,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,80.275390625,80.275390625,80.275390625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,85.5234375,85.5234375,85.5234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,101.0234375,101.0234375,101.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,150.0234375,150.0234375,150.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,320.0234375,320.0234375,320.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,948.0234375,948.0234375,948.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,3356.0234375,3356.0234375,3356.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:07,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,80.28125,80.28125,80.28125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,84.5546875,84.5546875,84.5546875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,101.0859375,101.0859375,101.0859375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,158.2734375,158.2734375,158.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,369.0234375,369.0234375,369.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,1176.0234375,1176.0234375,1176.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,4332.0234375,4332.0234375,4332.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,103.0380859375,103.0380859375,103.0380859375,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,120.78515625,120.78515625,120.78515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,166.78515625,166.78515625,166.78515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,313.03515625,313.03515625,313.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,823.03515625,823.03515625,823.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,2707.03515625,2707.03515625,2707.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,9931.03515625,9931.03515625,9931.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:14,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,103.0419921875,103.0419921875,103.0419921875,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,117.05078125,117.05078125,117.05078125,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,167.34765625,167.34765625,167.34765625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,337.28515625,337.28515625,337.28515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,968.03515625,968.03515625,968.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,3383.03515625,3383.03515625,3383.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,12827.03515625,12827.03515625,12827.03515625,"{""batch_size"": 4, ""hidden_size"": 768, ""num_heads"": 12, ""kernel_size"": 7, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,122.55078125,122.55078125,122.55078125,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,131.046875,131.046875,131.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,162.046875,162.046875,162.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,260.046875,260.046875,260.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,600.046875,600.046875,600.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,1856.046875,1856.046875,1856.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,6672.046875,6672.046875,6672.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:22,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,122.5546875,122.5546875,122.5546875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,130.06640625,130.06640625,130.06640625,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,162.109375,162.109375,162.109375,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,276.296875,276.296875,276.296875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,697.046875,697.046875,697.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,2308.046875,2308.046875,2308.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,8608.046875,8608.046875,8608.046875,"{""batch_size"": 2, ""hidden_size"": 1024, ""num_heads"": 16, ""kernel_size"": 9, ""dilation"": 1, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,64,80.275390625,80.275390625,80.275390625,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,128,85.5234375,85.5234375,85.5234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,256,101.0234375,101.0234375,101.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,512,150.0234375,150.0234375,150.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,1024,320.0234375,320.0234375,320.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,2048,948.0234375,948.0234375,948.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,liger,full,memory,MB,seq_len,sequence length,4096,3356.0234375,3356.0234375,3356.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:32,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,64,80.28125,80.28125,80.28125,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,128,84.5546875,84.5546875,84.5546875,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,256,101.0859375,101.0859375,101.0859375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,512,158.2734375,158.2734375,158.2734375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,1024,369.0234375,369.0234375,369.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,2048,1176.0234375,1176.0234375,1176.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +fused_neighborhood_attention,torch,full,memory,MB,seq_len,sequence length,4096,4332.0234375,4332.0234375,4332.0234375,"{""batch_size"": 2, ""hidden_size"": 512, ""num_heads"": 8, ""kernel_size"": 7, ""dilation"": 2, ""bias"": true, ""dtype"": ""torch.float32""}",NVIDIA H100 80GB HBM3,2025-05-27 19:27:39,0.5.10 +distill_cosine_loss,liger,forward,speed,ms,BT,B x T,1024,13.828096389770508,13.821133041381836,13.885849952697754,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:19:52,0.5.10 +distill_cosine_loss,liger,forward,speed,ms,BT,B x T,2048,27.57427215576172,27.52573432922363,27.579801940917967,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:19:52,0.5.10 +distill_cosine_loss,liger,forward,speed,ms,BT,B x T,4096,54.79423904418945,54.79423904418945,54.79423904418945,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:19:52,0.5.10 +distill_cosine_loss,liger,forward,speed,ms,BT,B x T,8192,109.73490905761719,109.73490905761719,109.73490905761719,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:19:52,0.5.10 +distill_cosine_loss,torch,forward,speed,ms,BT,B x T,1024,16.456703186035156,15.045836448669434,16.761650466918944,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:20:34,0.5.10 +distill_cosine_loss,torch,forward,speed,ms,BT,B x T,2048,29.703168869018555,29.69333839416504,29.71177024841309,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:20:34,0.5.10 +distill_cosine_loss,torch,forward,speed,ms,BT,B x T,4096,59.177982330322266,59.177982330322266,59.177982330322266,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:20:34,0.5.10 +distill_cosine_loss,torch,forward,speed,ms,BT,B x T,8192,118.3815689086914,118.3815689086914,118.3815689086914,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:20:34,0.5.10 +distill_cosine_loss,liger,full,speed,ms,BT,B x T,1024,14.654463768005371,14.63398380279541,14.68006420135498,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:21:16,0.5.10 +distill_cosine_loss,liger,full,speed,ms,BT,B x T,2048,28.274688720703125,28.27284507751465,28.279603958129883,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:21:16,0.5.10 +distill_cosine_loss,liger,full,speed,ms,BT,B x T,4096,55.96672058105469,55.96672058105469,55.96672058105469,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:21:16,0.5.10 +distill_cosine_loss,liger,full,speed,ms,BT,B x T,8192,111.38764953613281,111.38764953613281,111.38764953613281,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:21:16,0.5.10 +distill_cosine_loss,torch,full,speed,ms,BT,B x T,1024,37.45382308959961,37.42556076049805,37.482085418701175,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:01,0.5.10 +distill_cosine_loss,torch,full,speed,ms,BT,B x T,2048,73.56620788574219,73.56620788574219,73.56620788574219,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:01,0.5.10 +distill_cosine_loss,torch,full,speed,ms,BT,B x T,4096,145.73056030273438,145.73056030273438,145.73056030273438,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:01,0.5.10 +distill_cosine_loss,torch,full,speed,ms,BT,B x T,8192,291.5000305175781,291.5000305175781,291.5000305175781,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:01,0.5.10 +distill_cosine_loss,liger,full,memory,MB,BT,B x T,1024,5059.26806640625,5059.26806640625,5059.26806640625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:43,0.5.10 +distill_cosine_loss,liger,full,memory,MB,BT,B x T,2048,5087.27587890625,5087.27587890625,5087.27587890625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:43,0.5.10 +distill_cosine_loss,liger,full,memory,MB,BT,B x T,4096,5143.29150390625,5143.29150390625,5143.29150390625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:43,0.5.10 +distill_cosine_loss,liger,full,memory,MB,BT,B x T,8192,5255.32275390625,5255.32275390625,5255.32275390625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:22:43,0.5.10 +distill_cosine_loss,torch,full,memory,MB,BT,B x T,1024,7566.2822265625,7566.2822265625,7566.2822265625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:23:28,0.5.10 +distill_cosine_loss,torch,full,memory,MB,BT,B x T,2048,11590.3134765625,11590.3134765625,11590.3134765625,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:23:28,0.5.10 +distill_cosine_loss,torch,full,memory,MB,BT,B x T,4096,19654.375,19654.375,19654.375,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:23:28,0.5.10 +distill_cosine_loss,torch,full,memory,MB,BT,B x T,8192,35782.5,35782.5,35782.5,"{""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": false, ""weight_hard_loss"": 0.5, ""weight_soft_loss"": 0.5, ""ignore_index"": -100}",NVIDIA A100-SXM4-80GB,2025-06-27 09:23:28,0.5.10 +layer_norm,liger,forward,speed,ms,N,hidden size,1024,0.018848000094294548,0.018400000408291817,0.020102400332689285,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:11,0.6.0 +layer_norm,liger,forward,speed,ms,N,hidden size,2048,0.029152000322937965,0.02876799926161766,0.029823999851942062,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:11,0.6.0 +layer_norm,liger,forward,speed,ms,N,hidden size,4096,0.05104000121355057,0.05036799982190132,0.05177599936723709,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:11,0.6.0 +layer_norm,liger,forward,speed,ms,N,hidden size,8192,0.0947519987821579,0.09436800330877304,0.09507200121879578,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:11,0.6.0 +layer_norm,liger,forward,speed,ms,N,hidden size,16384,0.18476800620555878,0.18396799266338348,0.1852159947156906,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:11,0.6.0 +layer_norm,huggingface,forward,speed,ms,N,hidden size,1024,0.023584000766277313,0.023423999547958374,0.023840000852942467,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:14,0.6.0 +layer_norm,huggingface,forward,speed,ms,N,hidden size,2048,0.03734400123357773,0.03702399879693985,0.037811201065778746,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:14,0.6.0 +layer_norm,huggingface,forward,speed,ms,N,hidden size,4096,0.06617599725723267,0.06560000032186508,0.06678400188684464,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:14,0.6.0 +layer_norm,huggingface,forward,speed,ms,N,hidden size,8192,0.15267199277877808,0.15190400183200836,0.15347200632095337,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:14,0.6.0 +layer_norm,huggingface,forward,speed,ms,N,hidden size,16384,0.3067840039730072,0.3046143889427185,0.3081152021884918,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:14,0.6.0 +layer_norm,liger,backward,speed,ms,N,hidden size,1024,0.12006399780511856,0.11653760075569153,0.12467200309038162,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:16,0.6.0 +layer_norm,liger,backward,speed,ms,N,hidden size,2048,0.1207360029220581,0.1176128014922142,0.1256511986255646,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:16,0.6.0 +layer_norm,liger,backward,speed,ms,N,hidden size,4096,0.16630400717258453,0.16412800550460815,0.16838400065898895,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:16,0.6.0 +layer_norm,liger,backward,speed,ms,N,hidden size,8192,0.31279999017715454,0.31116798520088196,0.3145279884338379,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:16,0.6.0 +layer_norm,liger,backward,speed,ms,N,hidden size,16384,0.5776320099830627,0.5753471970558167,0.5798912048339844,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:16,0.6.0 +layer_norm,huggingface,backward,speed,ms,N,hidden size,1024,0.0605119988322258,0.059647999703884125,0.061344001442193985,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:18,0.6.0 +layer_norm,huggingface,backward,speed,ms,N,hidden size,2048,0.09967999905347824,0.09849599748849869,0.10099200159311295,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:18,0.6.0 +layer_norm,huggingface,backward,speed,ms,N,hidden size,4096,0.17881600558757782,0.17795200645923615,0.17971199750900269,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:18,0.6.0 +layer_norm,huggingface,backward,speed,ms,N,hidden size,8192,0.33369600772857666,0.3328000009059906,0.33478400111198425,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:18,0.6.0 +layer_norm,huggingface,backward,speed,ms,N,hidden size,16384,0.6424000263214111,0.6412223815917969,0.643455982208252,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:18,0.6.0 +layer_norm,liger,full,speed,ms,N,hidden size,1024,0.26576000452041626,0.2629248082637787,0.2701759934425354,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:21,0.6.0 +layer_norm,liger,full,speed,ms,N,hidden size,2048,0.27427199482917786,0.26999040842056277,0.28091518878936766,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:21,0.6.0 +layer_norm,liger,full,speed,ms,N,hidden size,4096,0.27454400062561035,0.27004799246788025,0.2807359993457794,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:21,0.6.0 +layer_norm,liger,full,speed,ms,N,hidden size,8192,0.40556800365448,0.40403199195861816,0.40723198652267456,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:21,0.6.0 +layer_norm,liger,full,speed,ms,N,hidden size,16384,0.7608960270881653,0.7589311957359314,0.7631679773330688,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:21,0.6.0 +layer_norm,huggingface,full,speed,ms,N,hidden size,1024,0.08025600016117096,0.07942400127649307,0.08111999928951263,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,speed,ms,N,hidden size,2048,0.13315199315547943,0.13180799782276154,0.13468800485134125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,speed,ms,N,hidden size,4096,0.2417600005865097,0.24089600145816803,0.24262399971485138,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,speed,ms,N,hidden size,8192,0.4832639992237091,0.48214399814605713,0.4843647956848145,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,speed,ms,N,hidden size,16384,0.950575977563858,0.9484800100326538,0.9528064012527466,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,liger,full,memory,MB,N,hidden size,1024,80.0625,80.0625,80.0625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,liger,full,memory,MB,N,hidden size,2048,160.09375,160.09375,160.09375,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,liger,full,memory,MB,N,hidden size,4096,320.15625,320.15625,320.15625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,liger,full,memory,MB,N,hidden size,8192,640.28125,640.28125,640.28125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,liger,full,memory,MB,N,hidden size,16384,1280.53125,1280.53125,1280.53125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,memory,MB,N,hidden size,1024,80.0625,80.0625,80.0625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,memory,MB,N,hidden size,2048,160.09375,160.09375,160.09375,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,memory,MB,N,hidden size,4096,320.15625,320.15625,320.15625,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,memory,MB,N,hidden size,8192,640.28125,640.28125,640.28125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +layer_norm,huggingface,full,memory,MB,N,hidden size,16384,1280.53125,1280.53125,1280.53125,"{""M"": 4096, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 80GB HBM3,2025-07-17 18:18:23,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,forward,speed,ms,H,hidden size,1024,0.01759999990463257,0.017311999574303627,0.017920000478625298,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:20,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,forward,speed,ms,H,hidden size,2048,0.02924799919128418,0.028863999992609024,0.029983999207615852,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:20,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,forward,speed,ms,H,hidden size,4096,0.05129599943757057,0.050624001771211624,0.05209600180387497,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:20,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,forward,speed,ms,H,hidden size,8192,0.09344000369310379,0.09296000003814697,0.09382399916648865,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:20,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,forward,speed,ms,H,hidden size,16384,0.1791680008172989,0.17814399302005768,0.1796800047159195,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:20,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,forward,speed,ms,H,hidden size,32768,0.43830400705337524,0.43744000792503357,0.43929600715637207,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:20,0.6.0 +fused_add_rms_norm,huggingface,forward,speed,ms,H,hidden size,1024,0.060095999389886856,0.059808000922203064,0.06054399907588959,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:23,0.6.0 +fused_add_rms_norm,huggingface,forward,speed,ms,H,hidden size,2048,0.09084799885749817,0.09027200192213058,0.09161599725484848,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:23,0.6.0 +fused_add_rms_norm,huggingface,forward,speed,ms,H,hidden size,4096,0.17820799350738525,0.17744000256061554,0.17897599935531616,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:23,0.6.0 +fused_add_rms_norm,huggingface,forward,speed,ms,H,hidden size,8192,0.312608003616333,0.3118720054626465,0.31324800848960876,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:23,0.6.0 +fused_add_rms_norm,huggingface,forward,speed,ms,H,hidden size,16384,0.574944019317627,0.5740479826927185,0.5756288051605225,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:23,0.6.0 +fused_add_rms_norm,huggingface,forward,speed,ms,H,hidden size,32768,1.0943039655685425,1.0934272289276123,1.0951999425888062,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:23,0.6.0 +fused_add_rms_norm,liger_rms_norm,forward,speed,ms,H,hidden size,1024,0.0352960005402565,0.03481600061058998,0.03811199963092804,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:26,0.6.0 +fused_add_rms_norm,liger_rms_norm,forward,speed,ms,H,hidden size,2048,0.05430399999022484,0.05392000079154968,0.05503999814391136,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:26,0.6.0 +fused_add_rms_norm,liger_rms_norm,forward,speed,ms,H,hidden size,4096,0.10592000186443329,0.1054655984044075,0.10630399733781815,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:26,0.6.0 +fused_add_rms_norm,liger_rms_norm,forward,speed,ms,H,hidden size,8192,0.19679999351501465,0.19631999731063843,0.19724799692630768,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:26,0.6.0 +fused_add_rms_norm,liger_rms_norm,forward,speed,ms,H,hidden size,16384,0.37436801195144653,0.3733760118484497,0.3752320110797882,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:26,0.6.0 +fused_add_rms_norm,liger_rms_norm,forward,speed,ms,H,hidden size,32768,0.7376000285148621,0.7361343741416931,0.7391359806060791,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:26,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,speed,ms,H,hidden size,1024,0.3147200047969818,0.30796160697937014,0.32764801383018494,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:30,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,speed,ms,H,hidden size,2048,0.3089919984340668,0.30374398827552795,0.3226880133152008,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:30,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,speed,ms,H,hidden size,4096,0.30691200494766235,0.3023296058177948,0.3205504059791565,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:30,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,speed,ms,H,hidden size,8192,0.3246079981327057,0.3185984075069428,0.33656961321830753,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:30,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,speed,ms,H,hidden size,16384,0.6010559797286987,0.5996800065040588,0.6026239991188049,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:30,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,speed,ms,H,hidden size,32768,1.8402559757232666,1.8322880268096924,1.8461120128631592,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:30,0.6.0 +fused_add_rms_norm,huggingface,full,speed,ms,H,hidden size,1024,0.23878400027751923,0.23545600473880768,0.2507520020008087,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:33,0.6.0 +fused_add_rms_norm,huggingface,full,speed,ms,H,hidden size,2048,0.34513600170612335,0.34377598762512207,0.34678399562835693,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:33,0.6.0 +fused_add_rms_norm,huggingface,full,speed,ms,H,hidden size,4096,0.6330879926681519,0.631712019443512,0.6345599889755249,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:33,0.6.0 +fused_add_rms_norm,huggingface,full,speed,ms,H,hidden size,8192,1.1185599565505981,1.1172800064086914,1.1196800470352173,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:33,0.6.0 +fused_add_rms_norm,huggingface,full,speed,ms,H,hidden size,16384,2.0697600841522217,2.0678528785705566,2.0713536739349365,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:33,0.6.0 +fused_add_rms_norm,huggingface,full,speed,ms,H,hidden size,32768,3.9561920166015625,3.953824043273926,3.9581120014190674,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:33,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,speed,ms,H,hidden size,1024,0.38916800916194916,0.3824320137500763,0.4037184059619903,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:36,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,speed,ms,H,hidden size,2048,0.3890720009803772,0.38193280100822447,0.4032831907272339,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:36,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,speed,ms,H,hidden size,4096,0.39715200662612915,0.3928639888763428,0.41097599267959595,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:36,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,speed,ms,H,hidden size,8192,0.6275200247764587,0.6259520053863525,0.6287999749183655,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:36,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,speed,ms,H,hidden size,16384,1.202239990234375,1.199679970741272,1.2048959732055664,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:36,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,speed,ms,H,hidden size,32768,2.7738559246063232,2.7705343723297116,2.777868890762329,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:36,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,backward,speed,ms,H,hidden size,1024,0.15619200468063354,0.15376000106334686,0.1661248028278351,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:39,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,backward,speed,ms,H,hidden size,2048,0.15825600177049637,0.15600000321865082,0.16911999881267548,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:39,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,backward,speed,ms,H,hidden size,4096,0.16700799763202667,0.16502399742603302,0.1709440052509308,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:39,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,backward,speed,ms,H,hidden size,8192,0.1712000072002411,0.1700800061225891,0.17215999960899353,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:39,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,backward,speed,ms,H,hidden size,16384,0.42505601048469543,0.4233280122280121,0.42691200971603394,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:39,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,backward,speed,ms,H,hidden size,32768,1.4057759642601013,1.3944000005722046,1.4099839925765991,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:39,0.6.0 +fused_add_rms_norm,huggingface,backward,speed,ms,H,hidden size,1024,0.1520960032939911,0.15136000514030457,0.1528960019350052,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:42,0.6.0 +fused_add_rms_norm,huggingface,backward,speed,ms,H,hidden size,2048,0.2533760070800781,0.2524160146713257,0.25436800718307495,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:42,0.6.0 +fused_add_rms_norm,huggingface,backward,speed,ms,H,hidden size,4096,0.4551039934158325,0.4540799856185913,0.45612800121307373,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:42,0.6.0 +fused_add_rms_norm,huggingface,backward,speed,ms,H,hidden size,8192,0.8053439855575562,0.8038079738616943,0.806656002998352,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:42,0.6.0 +fused_add_rms_norm,huggingface,backward,speed,ms,H,hidden size,16384,1.4933120012283325,1.492095947265625,1.49452805519104,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:42,0.6.0 +fused_add_rms_norm,huggingface,backward,speed,ms,H,hidden size,32768,2.8600640296936035,2.8583295822143557,2.8612607955932616,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:42,0.6.0 +fused_add_rms_norm,liger_rms_norm,backward,speed,ms,H,hidden size,1024,0.20175999402999878,0.199072003364563,0.2154303938150406,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,backward,speed,ms,H,hidden size,2048,0.20263999700546265,0.20000000298023224,0.21675519943237304,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,backward,speed,ms,H,hidden size,4096,0.25276800990104675,0.2515519857406616,0.2539199888706207,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,backward,speed,ms,H,hidden size,8192,0.4322720021009445,0.43088001012802124,0.4336000084877014,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,backward,speed,ms,H,hidden size,16384,0.8288000226020813,0.8266303777694701,0.8311295866966247,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,backward,speed,ms,H,hidden size,32768,2.03987193107605,2.0360767364501955,2.0436416149139403,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,memory,MB,H,hidden size,1024,72.546875,72.546875,72.546875,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,memory,MB,H,hidden size,2048,145.0859375,145.0859375,145.0859375,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,memory,MB,H,hidden size,4096,290.1640625,290.1640625,290.1640625,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,memory,MB,H,hidden size,8192,580.3203125,580.3203125,580.3203125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,memory,MB,H,hidden size,16384,1160.6328125,1160.6328125,1160.6328125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_fused_add_rms_norm,full,memory,MB,H,hidden size,32768,2321.2578125,2321.2578125,2321.2578125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,huggingface,full,memory,MB,H,hidden size,1024,104.03173828125,104.03173828125,104.03173828125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,huggingface,full,memory,MB,H,hidden size,2048,208.05517578125,208.05517578125,208.05517578125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,huggingface,full,memory,MB,H,hidden size,4096,416.10205078125,416.10205078125,416.10205078125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,huggingface,full,memory,MB,H,hidden size,8192,832.19580078125,832.19580078125,832.19580078125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,huggingface,full,memory,MB,H,hidden size,16384,1664.3125,1664.3125,1664.3125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,huggingface,full,memory,MB,H,hidden size,32768,3328.625,3328.625,3328.625,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,memory,MB,H,hidden size,1024,104.03564453125,104.03564453125,104.03564453125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,memory,MB,H,hidden size,2048,208.06298828125,208.06298828125,208.06298828125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,memory,MB,H,hidden size,4096,416.11767578125,416.11767578125,416.11767578125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,memory,MB,H,hidden size,8192,832.22705078125,832.22705078125,832.22705078125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,memory,MB,H,hidden size,16384,1544.44580078125,1544.44580078125,1544.44580078125,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_add_rms_norm,liger_rms_norm,full,memory,MB,H,hidden size,32768,2960.8837890625,2960.8837890625,2960.8837890625,"{""M"": 2048, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA H100 NVL,2025-07-16 07:04:45,0.6.0 +fused_linear_grpo_loss_token,liger,forward,speed,ms,B,B,2,40.75366401672363,40.749671173095706,40.75765686035156,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:58:45,0.6.1 +fused_linear_grpo_loss_token,liger,forward,speed,ms,B,B,4,80.95231628417969,80.95231628417969,80.95231628417969,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:58:45,0.6.1 +fused_linear_grpo_loss_token,liger,forward,speed,ms,B,B,8,163.58604431152344,163.58604431152344,163.58604431152344,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:58:45,0.6.1 +fused_linear_grpo_loss_token,liger,forward,speed,ms,B,B,16,323.6761474609375,323.6761474609375,323.6761474609375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:58:45,0.6.1 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,B,2,23.71225643157959,23.612825775146483,23.8354434967041,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:59:51,0.6.1 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,B,4,46.86131286621094,46.80355911254883,46.91906661987304,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:59:51,0.6.1 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,B,8,94.54898834228516,94.54898834228516,94.54898834228516,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:59:51,0.6.1 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,B,16,189.99501037597656,189.99501037597656,189.99501037597656,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-04 23:59:51,0.6.1 +fused_linear_grpo_loss_token,liger,full,speed,ms,B,B,2,42.67263984680176,42.54085083007813,42.80442886352539,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:00:58,0.6.1 +fused_linear_grpo_loss_token,liger,full,speed,ms,B,B,4,82.2446060180664,82.2446060180664,82.2446060180664,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:00:58,0.6.1 +fused_linear_grpo_loss_token,liger,full,speed,ms,B,B,8,167.00416564941406,167.00416564941406,167.00416564941406,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:00:58,0.6.1 +fused_linear_grpo_loss_token,liger,full,speed,ms,B,B,16,327.0911865234375,327.0911865234375,327.0911865234375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:00:58,0.6.1 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,B,2,45.36115264892578,45.241344451904304,45.480960845947266,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:02:07,0.6.1 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,B,4,90.00038146972656,90.00038146972656,90.00038146972656,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:02:07,0.6.1 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,B,8,177.22674560546875,177.22674560546875,177.22674560546875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:02:07,0.6.1 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,B,16,356.5383605957031,356.5383605957031,356.5383605957031,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:02:07,0.6.1 +fused_linear_grpo_loss_token,liger,backward,speed,ms,B,B,2,1.814527988433838,1.8124799728393555,1.8167808055877686,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:03:11,0.6.1 +fused_linear_grpo_loss_token,liger,backward,speed,ms,B,B,4,1.84934401512146,1.8472959995269775,1.8524160385131836,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:03:11,0.6.1 +fused_linear_grpo_loss_token,liger,backward,speed,ms,B,B,8,1.891327977180481,1.8872319459915161,1.893990397453308,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:03:11,0.6.1 +fused_linear_grpo_loss_token,liger,backward,speed,ms,B,B,16,1.9722239971160889,1.9660799503326416,1.9763200283050537,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:03:11,0.6.1 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,B,2,22.014975547790527,21.710438537597657,22.19417533874512,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:04:16,0.6.1 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,B,4,41.83603096008301,41.752165222167974,41.91989669799805,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:04:16,0.6.1 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,B,8,81.66400146484375,81.66400146484375,81.66400146484375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:04:16,0.6.1 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,B,16,162.6429443359375,162.6429443359375,162.6429443359375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:04:16,0.6.1 +fused_linear_grpo_loss_token,liger,full,memory,MB,B,B,2,7344.77685546875,7344.77685546875,7344.77685546875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:05:31,0.6.1 +fused_linear_grpo_loss_token,liger,full,memory,MB,B,B,4,7408.80029296875,7408.80029296875,7408.80029296875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:05:31,0.6.1 +fused_linear_grpo_loss_token,liger,full,memory,MB,B,B,8,7536.84716796875,7536.84716796875,7536.84716796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:05:31,0.6.1 +fused_linear_grpo_loss_token,liger,full,memory,MB,B,B,16,7792.94091796875,7792.94091796875,7792.94091796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:05:31,0.6.1 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,B,2,9083.28125,9083.28125,9083.28125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:06:37,0.6.1 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,B,4,13138.3125,13138.3125,13138.3125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:06:37,0.6.1 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,B,8,21250.375,21250.375,21250.375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:06:37,0.6.1 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,B,16,37474.5,37474.5,37474.5,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""token"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:06:37,0.6.1 +fused_linear_grpo_loss_sequence,liger,forward,speed,ms,B,B,2,40.72038269042969,40.71178131103516,40.728984069824214,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:07:48,0.6.1 +fused_linear_grpo_loss_sequence,liger,forward,speed,ms,B,B,4,81.69369506835938,81.69369506835938,81.69369506835938,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:07:48,0.6.1 +fused_linear_grpo_loss_sequence,liger,forward,speed,ms,B,B,8,162.79653930664062,162.79653930664062,162.79653930664062,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:07:48,0.6.1 +fused_linear_grpo_loss_sequence,liger,forward,speed,ms,B,B,16,323.6546630859375,323.6546630859375,323.6546630859375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:07:48,0.6.1 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,B,2,23.70047950744629,23.628594589233398,23.732429122924806,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:08:54,0.6.1 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,B,4,47.36921691894531,47.085364532470706,47.65306930541992,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:08:54,0.6.1 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,B,8,94.83366394042969,94.83366394042969,94.83366394042969,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:08:54,0.6.1 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,B,16,190.0963897705078,190.0963897705078,190.0963897705078,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:08:54,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,speed,ms,B,B,2,42.318336486816406,42.15214080810547,42.48453216552734,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:10:02,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,speed,ms,B,B,4,82.4616928100586,82.4616928100586,82.4616928100586,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:10:02,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,speed,ms,B,B,8,163.43756103515625,163.43756103515625,163.43756103515625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:10:02,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,speed,ms,B,B,16,325.4384765625,325.4384765625,325.4384765625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:10:02,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,B,2,45.99193572998047,45.80761489868165,46.176256561279295,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:11:10,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,B,4,88.57190704345703,88.57190704345703,88.57190704345703,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:11:10,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,B,8,176.94105529785156,176.94105529785156,176.94105529785156,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:11:10,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,B,16,356.0478820800781,356.0478820800781,356.0478820800781,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:11:10,0.6.1 +fused_linear_grpo_loss_sequence,liger,backward,speed,ms,B,B,2,1.8242560029029846,1.8102271556854248,1.8309119939804077,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:12:14,0.6.1 +fused_linear_grpo_loss_sequence,liger,backward,speed,ms,B,B,4,1.84934401512146,1.846886396408081,1.8534400463104248,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:12:14,0.6.1 +fused_linear_grpo_loss_sequence,liger,backward,speed,ms,B,B,8,1.891327977180481,1.8892799615859985,1.8933759927749634,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:12:14,0.6.1 +fused_linear_grpo_loss_sequence,liger,backward,speed,ms,B,B,16,1.9752960205078125,1.9722239971160889,1.977344036102295,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:12:14,0.6.1 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,B,2,22.0262393951416,21.80997085571289,22.20482559204102,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:13:20,0.6.1 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,B,4,41.54521560668945,41.224806213378905,41.865625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:13:20,0.6.1 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,B,8,81.21753692626953,81.21753692626953,81.21753692626953,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:13:20,0.6.1 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,B,16,160.82022094726562,160.82022094726562,160.82022094726562,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:13:20,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,memory,MB,B,B,2,7344.77685546875,7344.77685546875,7344.77685546875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:14:28,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,memory,MB,B,B,4,7408.80029296875,7408.80029296875,7408.80029296875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:14:28,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,memory,MB,B,B,8,7536.84716796875,7536.84716796875,7536.84716796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:14:28,0.6.1 +fused_linear_grpo_loss_sequence,liger,full,memory,MB,B,B,16,7792.94091796875,7792.94091796875,7792.94091796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:14:28,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,B,2,9083.28125,9083.28125,9083.28125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:15:31,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,B,4,13138.3125,13138.3125,13138.3125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:15:31,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,B,8,21250.375,21250.375,21250.375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:15:31,0.6.1 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,B,16,37474.5,37474.5,37474.5,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""importance_sampling_level"": ""sequence"", ""dtype"": ""torch.bfloat16""}",NVIDIA A100-SXM4-80GB,2025-08-05 00:15:31,0.6.1 +llama4_rope,liger,forward,speed,ms,H,hidden size,512,0.08249600231647491,0.08102399855852127,0.08432000130414963,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:01,0.6.1 +llama4_rope,liger,forward,speed,ms,H,hidden size,2048,0.08169600367546082,0.08037760108709335,0.08329600095748901,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:01,0.6.1 +llama4_rope,liger,forward,speed,ms,H,hidden size,8192,0.08128000050783157,0.07980799674987793,0.08329600095748901,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:01,0.6.1 +llama4_rope,huggingface,forward,speed,ms,H,hidden size,512,0.03759999945759773,0.03612799942493439,0.03907199949026108,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:03,0.6.1 +llama4_rope,huggingface,forward,speed,ms,H,hidden size,2048,0.06185600161552429,0.061267200857400894,0.06252799928188324,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:03,0.6.1 +llama4_rope,huggingface,forward,speed,ms,H,hidden size,8192,0.206496000289917,0.20582400262355804,0.20716799795627594,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:03,0.6.1 +llama4_rope,liger,backward,speed,ms,H,hidden size,512,0.15404799580574036,0.15241600573062897,0.15615999698638916,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:04,0.6.1 +llama4_rope,liger,backward,speed,ms,H,hidden size,2048,0.1536320000886917,0.15190400183200836,0.1558080017566681,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:04,0.6.1 +llama4_rope,liger,backward,speed,ms,H,hidden size,8192,0.15263999998569489,0.15094399452209473,0.15491199493408203,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:04,0.6.1 +llama4_rope,huggingface,backward,speed,ms,H,hidden size,512,0.13760000467300415,0.13574400544166565,0.14009599387645721,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:05,0.6.1 +llama4_rope,huggingface,backward,speed,ms,H,hidden size,2048,0.13600000739097595,0.13449600338935852,0.1382720023393631,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:05,0.6.1 +llama4_rope,huggingface,backward,speed,ms,H,hidden size,8192,0.21011200547218323,0.20924800634384155,0.21110400557518005,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:05,0.6.1 +llama4_rope,liger,full,speed,ms,H,hidden size,512,0.3652159869670868,0.3619840145111084,0.3699840009212494,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:07,0.6.1 +llama4_rope,liger,full,speed,ms,H,hidden size,2048,0.3599040061235428,0.2881920039653778,0.36559998989105225,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:07,0.6.1 +llama4_rope,liger,full,speed,ms,H,hidden size,8192,0.2874239981174469,0.2852480113506317,0.29029120206832887,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:07,0.6.1 +llama4_rope,huggingface,full,speed,ms,H,hidden size,512,0.24691200256347656,0.24489599466323853,0.24961919784545897,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,huggingface,full,speed,ms,H,hidden size,2048,0.24774399399757385,0.24582399427890778,0.2505407989025116,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,huggingface,full,speed,ms,H,hidden size,8192,0.41414400935173035,0.41337600350379944,0.41491198539733887,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,liger,full,memory,MB,H,hidden size,512,37.23486328125,37.23486328125,37.23486328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,liger,full,memory,MB,H,hidden size,2048,52.89111328125,52.89111328125,52.89111328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,liger,full,memory,MB,H,hidden size,8192,115.51611328125,115.51611328125,115.51611328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,huggingface,full,memory,MB,H,hidden size,512,49.64111328125,49.64111328125,49.64111328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,huggingface,full,memory,MB,H,hidden size,2048,102.51611328125,102.51611328125,102.51611328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,huggingface,full,memory,MB,H,hidden size,8192,314.01611328125,314.01611328125,314.01611328125,"{""dtype"": ""torch.bfloat16"", ""seq_len"": 2048, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:08,0.6.1 +llama4_rope,liger,forward,speed,ms,T,sequence length,1024,0.07417599856853485,0.07248000055551529,0.07596799731254578,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:10,0.6.1 +llama4_rope,liger,forward,speed,ms,T,sequence length,2048,0.08182399719953537,0.08006399869918823,0.08380799740552902,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:10,0.6.1 +llama4_rope,liger,forward,speed,ms,T,sequence length,4096,0.11708799749612808,0.1167680025100708,0.11744000017642975,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:10,0.6.1 +llama4_rope,liger,forward,speed,ms,T,sequence length,8192,0.2165440022945404,0.21596799790859222,0.21715199947357178,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:10,0.6.1 +llama4_rope,liger,forward,speed,ms,T,sequence length,16384,0.41756799817085266,0.41705599427223206,0.41811200976371765,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:10,0.6.1 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,1024,0.11644800007343292,0.11590400338172913,0.11708799749612808,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:12,0.6.1 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,2048,0.20659199357032776,0.20608000457286835,0.2072640061378479,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:12,0.6.1 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,4096,0.38553598523139954,0.3846847891807556,0.38624000549316406,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:12,0.6.1 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,8192,0.7411519885063171,0.7403839826583862,0.7420480251312256,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:12,0.6.1 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,16384,1.4553920030593872,1.4543871641159059,1.4562879800796509,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:12,0.6.1 +llama4_rope,liger,backward,speed,ms,T,sequence length,1024,0.11840000003576279,0.11711999773979187,0.12031999975442886,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:15,0.6.1 +llama4_rope,liger,backward,speed,ms,T,sequence length,2048,0.12336000055074692,0.12198399752378464,0.12489599734544754,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:15,0.6.1 +llama4_rope,liger,backward,speed,ms,T,sequence length,4096,0.12380799651145935,0.12240000069141388,0.12559999525547028,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:15,0.6.1 +llama4_rope,liger,backward,speed,ms,T,sequence length,8192,0.2170879989862442,0.2165759950876236,0.21753600239753723,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:15,0.6.1 +llama4_rope,liger,backward,speed,ms,T,sequence length,16384,0.4175359904766083,0.41705599427223206,0.4181375920772552,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:15,0.6.1 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,1024,0.1189119964838028,0.11769600212574005,0.12003199756145477,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:17,0.6.1 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,2048,0.21011200547218323,0.20927999913692474,0.21119999885559082,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:17,0.6.1 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,4096,0.39740800857543945,0.3963199853897095,0.39824000000953674,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:17,0.6.1 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,8192,0.7540159821510315,0.7528960108757019,0.7550719976425171,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:17,0.6.1 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,16384,1.4822720289230347,1.4810559749603271,1.4833600521087646,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:17,0.6.1 +llama4_rope,liger,full,speed,ms,T,sequence length,1024,0.2874400019645691,0.2853440046310425,0.29052799940109253,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:19,0.6.1 +llama4_rope,liger,full,speed,ms,T,sequence length,2048,0.28646400570869446,0.2845759987831116,0.28963199257850647,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:19,0.6.1 +llama4_rope,liger,full,speed,ms,T,sequence length,4096,0.29897600412368774,0.29660800099372864,0.302131199836731,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:19,0.6.1 +llama4_rope,liger,full,speed,ms,T,sequence length,8192,0.4315840005874634,0.4304639995098114,0.43270400166511536,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:19,0.6.1 +llama4_rope,liger,full,speed,ms,T,sequence length,16384,0.833184003829956,0.8322240114212036,0.8345024228096007,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:19,0.6.1 +llama4_rope,huggingface,full,speed,ms,T,sequence length,1024,0.24592000246047974,0.24396799504756927,0.24876800179481506,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,speed,ms,T,sequence length,2048,0.4138239920139313,0.41308799386024475,0.4145599901676178,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,speed,ms,T,sequence length,4096,0.7800959944725037,0.7790719866752625,0.7810239791870117,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,speed,ms,T,sequence length,8192,1.4911680221557617,1.4902976036071778,1.4922879934310913,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,speed,ms,T,sequence length,16384,2.9344160556793213,2.9333438873291016,2.9353599548339844,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,liger,full,memory,MB,T,sequence length,1024,73.75830078125,73.75830078125,73.75830078125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,liger,full,memory,MB,T,sequence length,2048,115.51611328125,115.51611328125,115.51611328125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,liger,full,memory,MB,T,sequence length,4096,199.03173828125,199.03173828125,199.03173828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,liger,full,memory,MB,T,sequence length,8192,366.06298828125,366.06298828125,366.06298828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,liger,full,memory,MB,T,sequence length,16384,700.12548828125,700.12548828125,700.12548828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,memory,MB,T,sequence length,1024,173.00830078125,173.00830078125,173.00830078125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,memory,MB,T,sequence length,2048,314.01611328125,314.01611328125,314.01611328125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,memory,MB,T,sequence length,4096,596.03173828125,596.03173828125,596.03173828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,memory,MB,T,sequence length,8192,1160.06298828125,1160.06298828125,1160.06298828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +llama4_rope,huggingface,full,memory,MB,T,sequence length,16384,2288.12548828125,2288.12548828125,2288.12548828125,"{""dtype"": ""torch.bfloat16"", ""hidden_size"": 8192, ""num_q_heads"": 32, ""num_kv_heads"": 8}",NVIDIA H100 80GB HBM3,2025-08-07 21:42:21,0.6.1 +tiled_geglu,liger,full,speed,ms,T,sequence length,1024,2.1678080558776855,2.166579246520996,2.1682305335998535,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:48,0.6.3 +tiled_geglu,liger,full,speed,ms,T,sequence length,2048,4.344256401062012,4.343987464904785,4.34452486038208,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:48,0.6.3 +tiled_geglu,liger,full,speed,ms,T,sequence length,4096,8.653023719787598,8.653023719787598,8.653023719787598,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:48,0.6.3 +tiled_geglu,liger,full,speed,ms,T,sequence length,8192,16.909311294555664,16.909311294555664,16.909311294555664,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:48,0.6.3 +tiled_geglu,liger,full,speed,ms,T,sequence length,16384,33.63123321533203,33.63123321533203,33.63123321533203,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:48,0.6.3 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,1024,3.353935956954956,3.353523015975952,3.35434889793396,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:49,0.6.3 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,2048,6.023168087005615,6.023168087005615,6.023168087005615,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:49,0.6.3 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,4096,11.495424270629883,11.495424270629883,11.495424270629883,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:49,0.6.3 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,8192,23.68614387512207,23.68614387512207,23.68614387512207,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:49,0.6.3 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,16384,47.478782653808594,47.478782653808594,47.478782653808594,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:49,0.6.3 +tiled_geglu,liger,forward,speed,ms,T,sequence length,1024,0.6614400148391724,0.6594560146331787,0.6635519862174988,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,liger,forward,speed,ms,T,sequence length,2048,1.3471999168395996,1.346560001373291,1.3475840091705322,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,liger,forward,speed,ms,T,sequence length,4096,2.752511978149414,2.7261502742767334,2.7844607830047607,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,liger,forward,speed,ms,T,sequence length,8192,5.433343887329102,5.433343887329102,5.433343887329102,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,liger,forward,speed,ms,T,sequence length,16384,10.712063789367676,10.712063789367676,10.712063789367676,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,1024,0.7403519749641418,0.7402047514915466,0.7413759827613831,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:53,0.6.3 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,2048,1.3941760063171387,1.3895679712295532,1.398144006729126,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:53,0.6.3 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,4096,2.7586560249328613,2.7585408687591553,2.759884834289551,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:53,0.6.3 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,8192,5.789696216583252,5.789696216583252,5.789696216583252,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:53,0.6.3 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,16384,11.810815811157227,11.810815811157227,11.810815811157227,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:53,0.6.3 +tiled_geglu,liger,backward,speed,ms,T,sequence length,1024,1.491968035697937,1.4916608333587646,1.4940160512924194,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:56,0.6.3 +tiled_geglu,liger,backward,speed,ms,T,sequence length,2048,3.0185279846191406,3.0131328105926514,3.0555264949798584,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:56,0.6.3 +tiled_geglu,liger,backward,speed,ms,T,sequence length,4096,6.021120071411133,6.021120071411133,6.021120071411133,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:56,0.6.3 +tiled_geglu,liger,backward,speed,ms,T,sequence length,8192,11.512767791748047,11.512767791748047,11.512767791748047,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:56,0.6.3 +tiled_geglu,liger,backward,speed,ms,T,sequence length,16384,22.806528091430664,22.806528091430664,22.806528091430664,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:56,0.6.3 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,1024,2.6060800552368164,2.6053311824798584,2.607308864593506,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:57,0.6.3 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,2048,4.665375709533691,4.664742469787598,4.666009426116943,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:57,0.6.3 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,4096,8.71731185913086,8.71731185913086,8.71731185913086,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:57,0.6.3 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,8192,17.99782371520996,17.99782371520996,17.99782371520996,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:57,0.6.3 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,16384,35.64400100708008,35.64400100708008,35.64400100708008,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:57,0.6.3 +tiled_geglu,liger,full,memory,MB,T,sequence length,1024,232.25,232.25,232.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger,full,memory,MB,T,sequence length,2048,336.25,336.25,336.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger,full,memory,MB,T,sequence length,4096,544.25,544.25,544.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger,full,memory,MB,T,sequence length,8192,960.25,960.25,960.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger,full,memory,MB,T,sequence length,16384,1792.25,1792.25,1792.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,1024,186.25,186.25,186.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,2048,244.25,244.25,244.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,4096,360.25,360.25,360.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,8192,592.25,592.25,592.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,16384,1056.25,1056.25,1056.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:00,0.6.3 +tiled_geglu,liger,forward,memory,MB,T,sequence length,1024,128.25,128.25,128.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:03,0.6.3 +tiled_geglu,liger,forward,memory,MB,T,sequence length,2048,192.25,192.25,192.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:03,0.6.3 +tiled_geglu,liger,forward,memory,MB,T,sequence length,4096,320.25,320.25,320.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:03,0.6.3 +tiled_geglu,liger,forward,memory,MB,T,sequence length,8192,576.25,576.25,576.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:03,0.6.3 +tiled_geglu,liger,forward,memory,MB,T,sequence length,16384,1088.25,1088.25,1088.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:03,0.6.3 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,1024,92.25,92.25,92.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,2048,120.25,120.25,120.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,4096,176.25,176.25,176.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,8192,288.25,288.25,288.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,16384,512.25,512.25,512.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,liger,backward,memory,MB,T,sequence length,1024,232.25,232.25,232.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,liger,backward,memory,MB,T,sequence length,2048,336.25,336.25,336.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,liger,backward,memory,MB,T,sequence length,4096,544.25,544.25,544.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,liger,backward,memory,MB,T,sequence length,8192,960.25,960.25,960.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,liger,backward,memory,MB,T,sequence length,16384,1792.25,1792.25,1792.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,1024,186.25,186.25,186.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:06,0.6.3 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,2048,244.25,244.25,244.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:06,0.6.3 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,4096,360.25,360.25,360.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:06,0.6.3 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,8192,592.25,592.25,592.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:06,0.6.3 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,16384,1056.25,1056.25,1056.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:06,0.6.3 +tiled_swiglu,liger,full,speed,ms,T,sequence length,1024,2.165760040283203,2.164659261703491,2.167193651199341,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:10,0.6.3 +tiled_swiglu,liger,full,speed,ms,T,sequence length,2048,4.371456146240234,4.368383884429932,4.374527931213379,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:10,0.6.3 +tiled_swiglu,liger,full,speed,ms,T,sequence length,4096,8.935423851013184,8.935423851013184,8.935423851013184,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:10,0.6.3 +tiled_swiglu,liger,full,speed,ms,T,sequence length,8192,17.078943252563477,17.078943252563477,17.078943252563477,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:10,0.6.3 +tiled_swiglu,liger,full,speed,ms,T,sequence length,16384,33.74857711791992,33.74857711791992,33.74857711791992,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:10,0.6.3 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,1024,3.3510398864746094,3.3507328033447266,3.3513472080230713,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:11,0.6.3 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,2048,6.023168087005615,6.023168087005615,6.023168087005615,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:11,0.6.3 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,4096,11.609087944030762,11.609087944030762,11.609087944030762,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:11,0.6.3 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,8192,23.8591365814209,23.8591365814209,23.8591365814209,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:11,0.6.3 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,16384,47.721473693847656,47.721473693847656,47.721473693847656,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:11,0.6.3 +tiled_swiglu,liger,forward,speed,ms,T,sequence length,1024,0.6594560146331787,0.6594560146331787,0.6604800224304199,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:14,0.6.3 +tiled_swiglu,liger,forward,speed,ms,T,sequence length,2048,1.3537280559539795,1.3527040481567383,1.3547519445419312,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:14,0.6.3 +tiled_swiglu,liger,forward,speed,ms,T,sequence length,4096,2.7152960300445557,2.715123176574707,2.7155072689056396,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:14,0.6.3 +tiled_swiglu,liger,forward,speed,ms,T,sequence length,8192,5.3361921310424805,5.3361921310424805,5.3361921310424805,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:14,0.6.3 +tiled_swiglu,liger,forward,speed,ms,T,sequence length,16384,10.870783805847168,10.870783805847168,10.870783805847168,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:14,0.6.3 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,1024,0.7395360469818115,0.7383040189743042,0.7413759827613831,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,2048,1.3965599536895752,1.387935996055603,1.4024640321731567,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,4096,2.7778561115264893,2.777395248413086,2.7780096530914307,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,8192,5.829631805419922,5.829631805419922,5.829631805419922,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,16384,11.841535568237305,11.841535568237305,11.841535568237305,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,liger,backward,speed,ms,T,sequence length,1024,1.4970879554748535,1.4961408376693726,1.4970879554748535,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:17,0.6.3 +tiled_swiglu,liger,backward,speed,ms,T,sequence length,2048,3.052351951599121,3.0518529415130615,3.0550782680511475,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:17,0.6.3 +tiled_swiglu,liger,backward,speed,ms,T,sequence length,4096,6.074687957763672,6.074687957763672,6.074687957763672,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:17,0.6.3 +tiled_swiglu,liger,backward,speed,ms,T,sequence length,8192,11.630592346191406,11.630592346191406,11.630592346191406,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:17,0.6.3 +tiled_swiglu,liger,backward,speed,ms,T,sequence length,16384,22.76793670654297,22.76793670654297,22.76793670654297,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:17,0.6.3 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,1024,2.6021440029144287,2.6000702381134033,2.6032767295837402,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:18,0.6.3 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,2048,4.641791820526123,4.641791820526123,4.641791820526123,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:18,0.6.3 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,4096,8.761343955993652,8.761343955993652,8.761343955993652,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:18,0.6.3 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,8192,17.966079711914062,17.966079711914062,17.966079711914062,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:18,0.6.3 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,16384,35.657344818115234,35.657344818115234,35.657344818115234,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:18,0.6.3 +tiled_swiglu,liger,full,memory,MB,T,sequence length,1024,232.25,232.25,232.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:21,0.6.3 +tiled_swiglu,liger,full,memory,MB,T,sequence length,2048,336.25,336.25,336.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:21,0.6.3 +tiled_swiglu,liger,full,memory,MB,T,sequence length,4096,544.25,544.25,544.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:21,0.6.3 +tiled_swiglu,liger,full,memory,MB,T,sequence length,8192,960.25,960.25,960.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:21,0.6.3 +tiled_swiglu,liger,full,memory,MB,T,sequence length,16384,1792.25,1792.25,1792.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:21,0.6.3 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,1024,186.25,186.25,186.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:22,0.6.3 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,2048,244.25,244.25,244.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:22,0.6.3 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,4096,360.25,360.25,360.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:22,0.6.3 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,8192,592.25,592.25,592.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:22,0.6.3 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,16384,1056.25,1056.25,1056.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:22,0.6.3 +tiled_swiglu,liger,forward,memory,MB,T,sequence length,1024,128.25,128.25,128.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger,forward,memory,MB,T,sequence length,2048,192.25,192.25,192.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger,forward,memory,MB,T,sequence length,4096,320.25,320.25,320.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger,forward,memory,MB,T,sequence length,8192,576.25,576.25,576.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger,forward,memory,MB,T,sequence length,16384,1088.25,1088.25,1088.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,1024,92.25,92.25,92.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,2048,120.25,120.25,120.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,4096,176.25,176.25,176.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,8192,288.25,288.25,288.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,16384,512.25,512.25,512.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:25,0.6.3 +tiled_swiglu,liger,backward,memory,MB,T,sequence length,1024,232.25,232.25,232.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:27,0.6.3 +tiled_swiglu,liger,backward,memory,MB,T,sequence length,2048,336.25,336.25,336.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:27,0.6.3 +tiled_swiglu,liger,backward,memory,MB,T,sequence length,4096,544.25,544.25,544.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:27,0.6.3 +tiled_swiglu,liger,backward,memory,MB,T,sequence length,8192,960.25,960.25,960.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:27,0.6.3 +tiled_swiglu,liger,backward,memory,MB,T,sequence length,16384,1792.25,1792.25,1792.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:27,0.6.3 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,1024,186.25,186.25,186.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:28,0.6.3 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,2048,244.25,244.25,244.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:28,0.6.3 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,4096,360.25,360.25,360.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:28,0.6.3 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,8192,592.25,592.25,592.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:28,0.6.3 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,16384,1056.25,1056.25,1056.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:28,0.6.3 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,1024,2.3357439041137695,2.3357439041137695,2.3375871181488037,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:47,0.6.3 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,2048,4.764671802520752,4.764671802520752,4.764671802520752,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:47,0.6.3 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,4096,9.4236478805542,9.4236478805542,9.4236478805542,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:47,0.6.3 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,8192,17.628543853759766,17.628543853759766,17.628543853759766,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:47,0.6.3 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,16384,35.06790542602539,35.06790542602539,35.06790542602539,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:47,0.6.3 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,1024,3.418976068496704,3.4176511764526367,3.4203009605407715,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:51,0.6.3 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,2048,6.158143997192383,6.158143997192383,6.158143997192383,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:51,0.6.3 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,4096,11.934720039367676,11.934720039367676,11.934720039367676,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:51,0.6.3 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,8192,24.731647491455078,24.731647491455078,24.731647491455078,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:51,0.6.3 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,16384,49.46227264404297,49.46227264404297,49.46227264404297,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:51,0.6.3 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,1024,0.6743040084838867,0.6736640334129333,0.677068829536438,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,2048,1.418239951133728,1.418239951133728,1.421120047569275,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,4096,2.88972806930542,2.889113664627075,2.8909568786621094,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,8192,5.701375961303711,5.701375961303711,5.701375961303711,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,16384,11.276288032531738,11.276288032531738,11.276288032531738,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:52,0.6.3 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,1024,0.7433919906616211,0.7423999905586243,0.7444480061531067,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:54,0.6.3 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,2048,1.4137760400772095,1.4131200313568115,1.4152319431304932,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:54,0.6.3 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,4096,2.8241920471191406,2.823500871658325,2.8266496658325195,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:54,0.6.3 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,8192,6.087679862976074,6.087679862976074,6.087679862976074,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:54,0.6.3 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,16384,12.353535652160645,12.353535652160645,12.353535652160645,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:54,0.6.3 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,1024,1.5499199628829956,1.5489535331726074,1.5523840188980103,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:55,0.6.3 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,2048,3.171328067779541,3.169484853744507,3.173171281814575,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:55,0.6.3 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,4096,6.263807773590088,6.263807773590088,6.263807773590088,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:55,0.6.3 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,8192,12.046143531799316,12.046143531799316,12.046143531799316,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:55,0.6.3 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,16384,23.839744567871094,23.839744567871094,23.839744567871094,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:55,0.6.3 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,1024,2.6757121086120605,2.6755776405334473,2.676710367202759,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:58,0.6.3 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,2048,4.7329277992248535,4.7329277992248535,4.7329277992248535,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:58,0.6.3 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,4096,9.078783988952637,9.078783988952637,9.078783988952637,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:58,0.6.3 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,8192,18.63680076599121,18.63680076599121,18.63680076599121,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:58,0.6.3 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,16384,37.06163024902344,37.06163024902344,37.06163024902344,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:58,0.6.3 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,1024,264.25,264.25,264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:59,0.6.3 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,2048,400.25,400.25,400.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:59,0.6.3 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,4096,688.25,688.25,688.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:59,0.6.3 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,8192,1264.25,1264.25,1264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:59,0.6.3 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,16384,2416.25,2416.25,2416.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:22:59,0.6.3 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,1024,190.25,190.25,190.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,2048,252.25,252.25,252.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,4096,376.25,376.25,376.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,8192,640.25,640.25,640.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,16384,1168.25,1168.25,1168.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,1024,144.25,144.25,144.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,2048,224.25,224.25,224.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,4096,384.25,384.25,384.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,8192,704.25,704.25,704.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,16384,1344.25,1344.25,1344.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:02,0.6.3 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,1024,90.25,90.25,90.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,2048,116.25,116.25,116.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,4096,168.25,168.25,168.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,8192,272.25,272.25,272.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,16384,480.25,480.25,480.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:04,0.6.3 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,1024,264.25,264.25,264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,2048,400.25,400.25,400.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,4096,688.25,688.25,688.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,8192,1264.25,1264.25,1264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,16384,2416.25,2416.25,2416.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:05,0.6.3 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,1024,190.25,190.25,190.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:07,0.6.3 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,2048,252.25,252.25,252.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:07,0.6.3 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,4096,376.25,376.25,376.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:07,0.6.3 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,8192,640.25,640.25,640.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:07,0.6.3 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,16384,1168.25,1168.25,1168.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""gelu_pytorch_tanh"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:07,0.6.3 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,1024,2.2517759799957275,2.2517759799957275,2.254848003387451,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:08,0.6.3 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,2048,4.588511943817139,4.587302207946777,4.5897216796875,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:08,0.6.3 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,4096,9.233407974243164,9.233407974243164,9.233407974243164,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:08,0.6.3 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,8192,17.869823455810547,17.869823455810547,17.869823455810547,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:08,0.6.3 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,16384,35.34422302246094,35.34422302246094,35.34422302246094,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:08,0.6.3 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,1024,3.4257922172546387,3.424870491027832,3.426713705062866,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:12,0.6.3 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,2048,6.155263900756836,6.155263900756836,6.155263900756836,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:12,0.6.3 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,4096,11.92959976196289,11.92959976196289,11.92959976196289,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:12,0.6.3 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,8192,24.815616607666016,24.815616607666016,24.815616607666016,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:12,0.6.3 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,16384,49.62918472290039,49.62918472290039,49.62918472290039,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:12,0.6.3 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,1024,0.6748160123825073,0.6737920045852661,0.6758400201797485,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:13,0.6.3 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,2048,1.4332799911499023,1.4325759410858154,1.4335999488830566,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:13,0.6.3 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,4096,2.91212797164917,2.904217481613159,2.9146623611450195,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:13,0.6.3 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,8192,5.658976078033447,5.658976078033447,5.658976078033447,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:13,0.6.3 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,16384,11.341952323913574,11.341952323913574,11.341952323913574,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:13,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,1024,0.7454720139503479,0.7429631948471069,0.7456768155097961,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,2048,1.4120960235595703,1.410048007965088,1.4120960235595703,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,4096,2.825216054916382,2.825216054916382,2.8264448642730713,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,8192,6.077439785003662,6.077439785003662,6.077439785003662,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,16384,12.356608390808105,12.356608390808105,12.356608390808105,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:15,0.6.3 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,1024,1.551360011100769,1.5511807203292847,1.5532032251358032,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:16,0.6.3 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,2048,3.1928319931030273,3.1885311603546143,3.1971328258514404,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:16,0.6.3 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,4096,6.273248195648193,6.273248195648193,6.273248195648193,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:16,0.6.3 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,8192,12.058752059936523,12.058752059936523,12.058752059936523,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:16,0.6.3 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,16384,23.853055953979492,23.853055953979492,23.853055953979492,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:16,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,1024,2.6746881008148193,2.6728639602661133,2.6789886951446533,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,2048,4.739071846008301,4.739071846008301,4.739071846008301,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,4096,9.084927558898926,9.084927558898926,9.084927558898926,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,8192,18.729759216308594,18.729759216308594,18.729759216308594,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,16384,37.13724899291992,37.13724899291992,37.13724899291992,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,1024,264.25,264.25,264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,2048,400.25,400.25,400.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,4096,688.25,688.25,688.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,8192,1264.25,1264.25,1264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,16384,2416.25,2416.25,2416.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:20,0.6.3 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,1024,190.25,190.25,190.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:23,0.6.3 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,2048,252.25,252.25,252.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:23,0.6.3 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,4096,376.25,376.25,376.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:23,0.6.3 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,8192,640.25,640.25,640.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:23,0.6.3 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,16384,1168.25,1168.25,1168.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:23,0.6.3 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,1024,144.25,144.25,144.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:24,0.6.3 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,2048,224.25,224.25,224.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:24,0.6.3 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,4096,384.25,384.25,384.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:24,0.6.3 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,8192,704.25,704.25,704.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:24,0.6.3 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,16384,1344.25,1344.25,1344.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:24,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,1024,90.25,90.25,90.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,2048,116.25,116.25,116.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,4096,168.25,168.25,168.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,8192,272.25,272.25,272.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,16384,480.25,480.25,480.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,1024,264.25,264.25,264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,2048,400.25,400.25,400.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,4096,688.25,688.25,688.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,8192,1264.25,1264.25,1264.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,16384,2416.25,2416.25,2416.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:26,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,1024,190.25,190.25,190.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:29,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,2048,252.25,252.25,252.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:29,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,4096,376.25,376.25,376.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:29,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,8192,640.25,640.25,640.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:29,0.6.3 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,16384,1168.25,1168.25,1168.25,"{""bsz"": 2, ""hidden_size"": 2048, ""intermediate_size"": 4096, ""hidden_act"": ""silu"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""dtype"": ""torch.bfloat16""}",NVIDIA GeForce RTX 4090,2025-11-11 06:23:29,0.6.3 +tvd,liger,full,memory,MB,V,vocab size,4096,1792.0009765625,1792.0009765625,1792.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,liger,full,memory,MB,V,vocab size,8192,3584.0009765625,3584.0009765625,3584.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,liger,full,memory,MB,V,vocab size,16384,7168.0009765625,7168.0009765625,7168.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,liger,full,memory,MB,V,vocab size,32768,14336.0009765625,14336.0009765625,14336.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,liger,full,memory,MB,V,vocab size,65536,28672.0,28672.0,28672.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,liger,full,memory,MB,V,vocab size,131072,57344.0,57344.0,57344.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,torch,full,memory,MB,V,vocab size,4096,2048.0009765625,2048.0009765625,2048.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,torch,full,memory,MB,V,vocab size,8192,4096.0009765625,4096.0009765625,4096.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,torch,full,memory,MB,V,vocab size,16384,8192.0009765625,8192.0009765625,8192.0009765625,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,torch,full,memory,MB,V,vocab size,32768,16384.0,16384.0,16384.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,torch,full,memory,MB,V,vocab size,65536,32768.0,32768.0,32768.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,torch,full,memory,MB,V,vocab size,131072,65536.0,65536.0,65536.0,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:48,0.7.0 +tvd,liger,forward,speed,ms,V,vocab size,4096,0.2757120132446289,0.27487359642982484,0.27616640329360964,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:50,0.7.0 +tvd,liger,forward,speed,ms,V,vocab size,8192,0.5338559746742249,0.5333759784698486,0.5346879959106445,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:50,0.7.0 +tvd,liger,forward,speed,ms,V,vocab size,16384,1.0511679649353027,1.0505280494689941,1.0521472215652465,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:50,0.7.0 +tvd,liger,forward,speed,ms,V,vocab size,32768,2.0986878871917725,2.09736967086792,2.0999168872833254,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:50,0.7.0 +tvd,liger,forward,speed,ms,V,vocab size,65536,4.221951961517334,4.22039680480957,4.222847938537598,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:50,0.7.0 +tvd,liger,forward,speed,ms,V,vocab size,131072,8.501215934753418,8.498592376708984,8.50380802154541,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:50,0.7.0 +tvd,torch,forward,speed,ms,V,vocab size,4096,0.7288320064544678,0.727942419052124,0.7296640276908875,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:51,0.7.0 +tvd,torch,forward,speed,ms,V,vocab size,8192,1.4264639616012573,1.42576003074646,1.4272960424423218,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:51,0.7.0 +tvd,torch,forward,speed,ms,V,vocab size,16384,2.81440007686615,2.8132031917572022,2.815097618103027,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:51,0.7.0 +tvd,torch,forward,speed,ms,V,vocab size,32768,5.5965118408203125,5.59548807144165,5.598131275177002,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:51,0.7.0 +tvd,torch,forward,speed,ms,V,vocab size,65536,11.178752422332764,11.176428604125977,11.180454635620118,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:51,0.7.0 +tvd,torch,forward,speed,ms,V,vocab size,131072,22.33670425415039,22.334880065917968,22.339027404785156,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:51,0.7.0 +tvd,liger,full,speed,ms,V,vocab size,4096,1.123952031135559,1.1221888303756713,1.1291328191757202,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:53,0.7.0 +tvd,liger,full,speed,ms,V,vocab size,8192,2.1660319566726685,2.162835216522217,2.169088077545166,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:53,0.7.0 +tvd,liger,full,speed,ms,V,vocab size,16384,4.563424110412598,4.559807777404785,4.5669121742248535,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:53,0.7.0 +tvd,liger,full,speed,ms,V,vocab size,32768,9.092079639434814,9.089529991149902,9.094182014465332,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:53,0.7.0 +tvd,liger,full,speed,ms,V,vocab size,65536,18.217248916625977,18.20675277709961,18.219014739990236,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:53,0.7.0 +tvd,liger,full,speed,ms,V,vocab size,131072,36.477935791015625,36.46965026855469,36.48622131347656,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:53,0.7.0 +tvd,torch,full,speed,ms,V,vocab size,4096,2.1256959438323975,2.1249279975891113,2.1270463466644287,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:54,0.7.0 +tvd,torch,full,speed,ms,V,vocab size,8192,4.191232204437256,4.189510250091553,4.192793464660644,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:54,0.7.0 +tvd,torch,full,speed,ms,V,vocab size,16384,8.638431549072266,8.636992454528809,8.639007568359375,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:54,0.7.0 +tvd,torch,full,speed,ms,V,vocab size,32768,17.25654411315918,17.25450286865234,17.25882225036621,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:54,0.7.0 +tvd,torch,full,speed,ms,V,vocab size,65536,34.54822540283203,34.546746826171876,34.549703979492186,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:54,0.7.0 +tvd,torch,full,speed,ms,V,vocab size,131072,69.17910766601562,69.17910766601562,69.17910766601562,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:54,0.7.0 +tvd,liger,backward,speed,ms,V,vocab size,4096,0.8502079844474792,0.8484799861907959,0.8526080250740051,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:55,0.7.0 +tvd,liger,backward,speed,ms,V,vocab size,8192,1.6321280002593994,1.629702377319336,1.6350399732589722,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:55,0.7.0 +tvd,liger,backward,speed,ms,V,vocab size,16384,3.5109760761260986,3.5084415912628173,3.513107109069824,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:55,0.7.0 +tvd,liger,backward,speed,ms,V,vocab size,32768,6.989071846008301,6.985472011566161,6.994240188598633,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:55,0.7.0 +tvd,liger,backward,speed,ms,V,vocab size,65536,13.969247817993164,13.95904598236084,13.971328163146971,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:55,0.7.0 +tvd,liger,backward,speed,ms,V,vocab size,131072,27.982528686523438,27.963673400878903,27.987577819824217,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:55,0.7.0 +tvd,torch,backward,speed,ms,V,vocab size,4096,1.398911952972412,1.3979583740234376,1.4000320434570312,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:57,0.7.0 +tvd,torch,backward,speed,ms,V,vocab size,8192,2.7701759338378906,2.7694976329803467,2.7718528747558593,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:57,0.7.0 +tvd,torch,backward,speed,ms,V,vocab size,16384,5.828160047531128,5.8249921798706055,5.829792022705078,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:57,0.7.0 +tvd,torch,backward,speed,ms,V,vocab size,32768,11.665760040283203,11.664883232116699,11.666317176818847,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:57,0.7.0 +tvd,torch,backward,speed,ms,V,vocab size,65536,23.379840850830078,23.37938575744629,23.381267929077147,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:57,0.7.0 +tvd,torch,backward,speed,ms,V,vocab size,131072,46.83844757080078,46.8328125,46.84408264160156,"{""B"": 8, ""T"": 2048}",NVIDIA H100 80GB HBM3,2026-03-03 23:02:57,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,32,0.017535999417304993,0.016863999888300896,0.01833599992096424,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,64,0.018848000094294548,0.018015999346971512,0.019487999379634857,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,128,0.026623999699950218,0.024607999250292778,0.026688000187277794,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,256,0.038943998515605927,0.03888000175356865,0.03903999924659729,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,512,0.06351999938488007,0.06345599889755249,0.06550399959087372,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,1024,0.11475200206041336,0.11468800157308578,0.11673600226640701,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,liger,forward,speed,ms,C,num_channels,2048,0.21910400688648224,0.217056006193161,0.22115199267864227,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:15,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,32,0.030688000842928886,0.030592000111937523,0.030751999467611313,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,64,0.043007999658584595,0.04294399917125702,0.04303999990224838,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,128,0.07168000191450119,0.07161600142717361,0.07174400240182877,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,256,0.13516800105571747,0.1351040005683899,0.13523200154304504,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,512,0.25808000564575195,0.2580159902572632,0.25900799036026,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,1024,0.4986239969730377,0.4976640045642853,0.4997439980506897,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,huggingface,forward,speed,ms,C,num_channels,2048,0.9819360077381134,0.9800639748573303,0.9830080270767212,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:19,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,32,0.1658720001578331,0.16368000209331512,0.16958080232143402,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,64,0.1730239987373352,0.17123199999332428,0.17520000040531158,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,128,0.1695999950170517,0.16783360242843628,0.1717183977365494,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,256,0.174112007021904,0.17206400632858276,0.17718400061130524,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,512,0.18745599687099457,0.18636800348758698,0.18848000466823578,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,1024,0.3388479948043823,0.33792001008987427,0.3400000035762787,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,liger,full,speed,ms,C,num_channels,2048,0.6390079855918884,0.6371200084686279,0.6410560011863708,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:22,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,32,0.08396799862384796,0.08390399813652039,0.08403199911117554,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,64,0.11267200112342834,0.11260800063610077,0.1128000020980835,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,128,0.20054399967193604,0.19868800044059753,0.20080000162124634,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,256,0.35020801424980164,0.34828799962997437,0.3511039912700653,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,512,0.6307839751243591,0.6297919750213623,0.6309120059013367,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,1024,1.177664041519165,1.1766079664230347,1.1796480417251587,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,huggingface,full,speed,ms,C,num_channels,2048,2.2947518825531006,2.292736053466797,2.296736001968384,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:26,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,32,0.06643199920654297,0.0655359998345375,0.06752000004053116,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,64,0.06732799857854843,0.0663679987192154,0.06838399916887283,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,128,0.07171200215816498,0.06969600170850754,0.07273600250482559,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,256,0.07580800354480743,0.07571200281381607,0.07683199644088745,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,512,0.12697599828243256,0.1249919980764389,0.12703999876976013,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,1024,0.2253440022468567,0.2252800017595291,0.22729599475860596,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,liger,backward,speed,ms,C,num_channels,2048,0.42585599422454834,0.42396798729896545,0.4260160028934479,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:28,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,32,0.05532800033688545,0.05526399984955788,0.056352000683546066,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,64,0.07372800260782242,0.07171200215816498,0.0739263966679573,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,128,0.13315199315547943,0.13308799266815186,0.13331200182437897,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,256,0.21916800737380981,0.21904000639915466,0.21926400065422058,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,512,0.374783992767334,0.37379199266433716,0.37484800815582275,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,1024,0.6820799708366394,0.6810240149497986,0.6839039921760559,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,backward,speed,ms,C,num_channels,2048,1.3158719539642334,1.3157440423965454,1.3177599906921387,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,32,40.01171875,40.01171875,40.01171875,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,64,80.01953125,80.01953125,80.01953125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,128,160.03515625,160.03515625,160.03515625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,256,320.0703125,320.0703125,320.0703125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,512,640.140625,640.140625,640.140625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,1024,1280.28125,1280.28125,1280.28125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,liger,full,memory,MB,C,num_channels,2048,2560.5625,2560.5625,2560.5625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:31,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,32,40.06640625,40.06640625,40.06640625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,64,80.12890625,80.12890625,80.12890625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,128,160.25390625,160.25390625,160.25390625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,256,320.5078125,320.5078125,320.5078125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,512,641.015625,641.015625,641.015625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,1024,1282.03125,1282.03125,1282.03125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,full,memory,MB,C,num_channels,2048,2564.0625,2564.0625,2564.0625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,32,40.01171875,40.01171875,40.01171875,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,64,80.01953125,80.01953125,80.01953125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,128,160.03515625,160.03515625,160.03515625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,256,320.0703125,320.0703125,320.0703125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,512,640.140625,640.140625,640.140625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,1024,1280.28125,1280.28125,1280.28125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,forward,memory,MB,C,num_channels,2048,2560.5625,2560.5625,2560.5625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,32,40.06640625,40.06640625,40.06640625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,64,80.12890625,80.12890625,80.12890625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,128,160.25390625,160.25390625,160.25390625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,256,320.5078125,320.5078125,320.5078125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,512,641.015625,641.015625,641.015625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,1024,1282.03125,1282.03125,1282.03125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,forward,memory,MB,C,num_channels,2048,2564.0625,2564.0625,2564.0625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,32,40.01171875,40.01171875,40.01171875,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,64,80.01953125,80.01953125,80.01953125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,128,160.03515625,160.03515625,160.03515625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,256,320.0703125,320.0703125,320.0703125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,512,640.140625,640.140625,640.140625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,1024,1280.28125,1280.28125,1280.28125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,liger,backward,memory,MB,C,num_channels,2048,2560.5625,2560.5625,2560.5625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,32,40.06640625,40.06640625,40.06640625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,64,80.12890625,80.12890625,80.12890625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,128,160.25390625,160.25390625,160.25390625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,256,320.5078125,320.5078125,320.5078125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,512,641.015625,641.015625,641.015625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,1024,1282.03125,1282.03125,1282.03125,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 +group_norm,huggingface,backward,memory,MB,C,num_channels,2048,2564.0625,2564.0625,2564.0625,"{""M"": 128, ""H"": 512, ""channels_per_group"": 4, ""dtype"": ""torch.float32"", ""eps"": 1e-06}",NVIDIA B200,2026-02-28 00:23:32,0.7.0 diff --git a/benchmark/scripts/__init__.py b/benchmark/scripts/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/benchmark/scripts/benchmark_cpo_loss.py b/benchmark/scripts/benchmark_cpo_loss.py new file mode 100755 index 0000000..8b10d51 --- /dev/null +++ b/benchmark/scripts/benchmark_cpo_loss.py @@ -0,0 +1,167 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +############################################################################# +# Test the memory consumption of the linear fused cross entropy loss +############################################################################# + + +def bench_memory_fused_linear_cpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_cpo_loss import LigerLMHeadCPO + from test.chunked_loss.test_cpo_loss import TorchLMHeadCPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + + # Instantiate once and retrieve the first output only + torch_lm_head_cpo = TorchLMHeadCPO(H=H, V=V, dtype=dtype).to(device) + liger_lm_head_cpo = LigerLMHeadCPO(H=H, V=V, dtype=dtype).to(device) + torch_fwd = lambda x, target: torch_lm_head_cpo(x, target)[0] + liger_fwd = lambda x, target: liger_lm_head_cpo(x, target)[0] + + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + def fwd(): + if provider == "liger": + return liger_fwd(_input, target) + elif provider == "huggingface": + return torch_fwd(_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +# ############################################################################# +# # Test the speed of the fused linear cross entropy loss +# ############################################################################# + + +def bench_speed_fused_linear_cpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_cpo_loss import LigerLMHeadCPO + from test.chunked_loss.test_cpo_loss import TorchLMHeadCPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + # Instantiate once and retrieve the first output only + torch_lm_head_cpo = TorchLMHeadCPO(H=H, V=V, dtype=dtype).to(device) + liger_lm_head_cpo = LigerLMHeadCPO(H=H, V=V, dtype=dtype).to(device) + torch_fwd = lambda x, target: torch_lm_head_cpo(x, target)[0] + liger_fwd = lambda x, target: liger_lm_head_cpo(x, target)[0] + + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + def fwd(): + if provider == "liger": + return liger_fwd(_input, target) + elif provider == "huggingface": + return torch_fwd(_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_linear_cpo_loss", + "x_name": "B", + "x_label": "B", + "x_values": [2**i for i in range(1, 5)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "T": 1024, + "H": 4096, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_cpo_loss, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_cpo_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_cross_entropy.py b/benchmark/scripts/benchmark_cross_entropy.py new file mode 100755 index 0000000..cdd6181 --- /dev/null +++ b/benchmark/scripts/benchmark_cross_entropy.py @@ -0,0 +1,126 @@ +import torch +import triton + +from torch.nn import CrossEntropyLoss +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.cross_entropy import LigerCrossEntropyLoss +from liger_kernel.utils import infer_device + +device = infer_device() + + +def bench_memory_cross_entropy( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + torch_ce = CrossEntropyLoss() + liger_ce = LigerCrossEntropyLoss() + + V = input.x + provider = input.kernel_provider + B = input.extra_benchmark_config["B"] + T = input.extra_benchmark_config["T"] + + _input = torch.randn(B * T, V, requires_grad=True, device=device) + target = torch.randint(V, (B * T, 1), device=device).squeeze(1) + + def fwd(): + if provider == "liger": + return liger_ce(_input, target) + else: + return torch_ce(_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +def bench_speed_cross_entropy( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + torch_ce = CrossEntropyLoss() + liger_ce = LigerCrossEntropyLoss() + + V = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + B = input.extra_benchmark_config["B"] + T = input.extra_benchmark_config["T"] + + _input = torch.randn(B * T, V, requires_grad=True, device=device) + target = torch.randint(V, (B * T, 1), device=device).squeeze(1) + + def fwd(): + if provider == "liger": + return liger_ce(_input, target) + else: + return torch_ce(_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, rep=100, quantiles=QUANTILES) + elif mode == "no-grad-forward": + with torch.no_grad(): + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, rep=100, quantiles=QUANTILES) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, rep=100, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "cross_entropy", + "x_name": "V", + "x_label": "vocab size", + "x_values": [2**i for i in range(12, 18)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [{"B": 8, "T": 2048}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_cross_entropy, + kernel_operation_modes=["forward", "backward", "full", "no-grad-forward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_cross_entropy, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_distill_cosine_loss.py b/benchmark/scripts/benchmark_distill_cosine_loss.py new file mode 100755 index 0000000..5cf12b4 --- /dev/null +++ b/benchmark/scripts/benchmark_distill_cosine_loss.py @@ -0,0 +1,266 @@ +import os +import sys + +import torch +import torch.nn as nn +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.chunked_loss.cosine_similarity_loss import LigerFusedLinearCosineSimilarityFunction +from liger_kernel.utils import infer_device + +device = infer_device() + +# Ensure the project root is in the path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +class TorchCosineSimilarityLoss(nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + weight_hard_loss: float = 0.5, + weight_soft_loss: float = 0.5, + ignore_index: int = -100, + temperature: float = 1.0, + bias: bool = False, + ): + from test.chunked_loss.test_cosine_loss import HFCosineLoss + + super().__init__() + self.student_lin = nn.Linear(in_features=H // 2, out_features=V, bias=bias).to(dtype=dtype) + self.teacher_lin = nn.Linear(in_features=H, out_features=V, bias=bias).to(dtype=dtype) + self.cosine_loss = HFCosineLoss( + ignore_index=ignore_index, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + temperature=temperature, + ).get_batch_loss_metrics + + def forward(self, student: torch.Tensor, teacher: torch.Tensor, target: torch.Tensor): + return self.cosine_loss(student, self.student_lin.weight, teacher, self.teacher_lin.weight, target) + + +class LigerCosineSimilarityLoss(nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + weight_hard_loss: float = 0.5, + weight_soft_loss: float = 0.5, + ignore_index: int = -100, + temperature: float = 1.0, + bias: bool = False, + ): + super().__init__() + self.student_lin = nn.Linear(in_features=H // 2, out_features=V, bias=bias).to(dtype=dtype) + self.teacher_lin = nn.Linear(in_features=H, out_features=V, bias=bias).to(dtype=dtype) + self.weight_hard_loss = weight_hard_loss + self.weight_soft_loss = weight_soft_loss + self.ignore_index = ignore_index + self.temperature = temperature + self.cosine_loss = LigerFusedLinearCosineSimilarityFunction.apply + + def forward(self, student: torch.Tensor, teacher: torch.Tensor, target: torch.Tensor): + return self.cosine_loss( + student, + self.student_lin.weight, + teacher, + self.teacher_lin.weight, + target, + self.student_lin.bias, + self.teacher_lin.bias, + self.weight_hard_loss, + self.weight_soft_loss, + ) + + +def bench_memory_cosine_similarity_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + weight_hard_loss = input.extra_benchmark_config["weight_hard_loss"] + weight_soft_loss = input.extra_benchmark_config["weight_soft_loss"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + + torch_cosine_loss = TorchCosineSimilarityLoss( + H=H, + V=V, + dtype=dtype, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + bias=bias, + ).to(device) + liger_cosine_loss = LigerCosineSimilarityLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + + _tensor = torch.rand(BT, H // 2, device=device, dtype=dtype) + student_input1 = _tensor.detach().clone().requires_grad_(True) + student_input2 = _tensor.detach().clone().requires_grad_(True) + + teacher_input = torch.rand(BT, H, device=device, dtype=dtype) + + target = torch.randint(0, V, (BT,), device=device, dtype=torch.long) + + def fwd(): + if provider == "liger": + return liger_cosine_loss(student_input1, teacher_input, target) + elif provider == "torch": + return torch_cosine_loss(student_input2, teacher_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +def bench_speed_cosine_similarity_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + weight_hard_loss = input.extra_benchmark_config["weight_hard_loss"] + weight_soft_loss = input.extra_benchmark_config["weight_soft_loss"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + torch_cosine_loss = TorchCosineSimilarityLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + + liger_cosine_loss = LigerCosineSimilarityLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + + _tensor = torch.rand(BT, H // 2, device=device, dtype=dtype) + student_input1 = _tensor.detach().clone().requires_grad_(True) + student_input2 = _tensor.detach().clone().requires_grad_(True) + + teacher_input = torch.rand(BT, H, device=device, dtype=dtype) + + target = torch.randint(0, V, (BT,), device=device, dtype=torch.long) + + def fwd(): + if provider == "liger": + return liger_cosine_loss(student_input1, teacher_input, target) + elif provider == "torch": + return torch_cosine_loss(student_input2, teacher_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[student_input1, student_input2], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "distill_cosine_loss", + "x_name": "BT", + "x_label": "B x T", + "x_values": [2**i for i in range(10, 14)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "H": 4096, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + "bias": False, + "weight_hard_loss": 0.5, + "weight_soft_loss": 0.5, + "ignore_index": -100, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_cosine_similarity_loss, + kernel_operation_modes=["forward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + + run_benchmarks( + bench_test_fn=bench_memory_cosine_similarity_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_distill_jsd_loss.py b/benchmark/scripts/benchmark_distill_jsd_loss.py new file mode 100755 index 0000000..324418e --- /dev/null +++ b/benchmark/scripts/benchmark_distill_jsd_loss.py @@ -0,0 +1,272 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.chunked_loss.jsd_loss import LigerFusedLinearJSDFunction +from liger_kernel.utils import get_total_gpu_memory +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +class TorchJSDLoss(torch.nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + weight_hard_loss: float = 0.5, + weight_soft_loss: float = 0.5, + ignore_index: int = -100, + temperature: float = 1.0, + bias: bool = False, + ): + from test.chunked_loss.test_jsd_loss import HFJSDLoss + + super().__init__() + self.student_lin = torch.nn.Linear(in_features=H // 2, out_features=V, bias=bias, dtype=dtype) + self.teacher_lin = torch.nn.Linear(in_features=H, out_features=V, bias=bias, dtype=dtype) + self.jsd_loss = HFJSDLoss( + ignore_index=ignore_index, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + temperature=temperature, + ).get_batch_loss_metrics + + def forward(self, student, teacher, target): + return self.jsd_loss( + student, + self.student_lin.weight, + teacher, + self.teacher_lin.weight, + target, + ) + + +class LigerJSDLoss(torch.nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + weight_hard_loss: float = 0.5, + weight_soft_loss: float = 0.5, + ignore_index: int = -100, + temperature: float = 1.0, + bias: bool = False, + ): + super().__init__() + self.student_lin = torch.nn.Linear(in_features=H // 2, out_features=V, bias=bias, dtype=dtype) + self.teacher_lin = torch.nn.Linear(in_features=H, out_features=V, bias=bias, dtype=dtype) + self.weight_hard_loss = weight_hard_loss + self.weight_soft_loss = weight_soft_loss + self.ignore_index = ignore_index + self.temperature = temperature + self.jsd_loss = LigerFusedLinearJSDFunction.apply + + def forward(self, student, teacher, target): + return self.jsd_loss( + student, + self.student_lin.weight, + teacher, + self.teacher_lin.weight, + target, + self.student_lin.bias, + self.teacher_lin.bias, + self.weight_hard_loss, + self.weight_soft_loss, + ) + + +def bench_memory_jsd_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + weight_hard_loss = input.extra_benchmark_config["weight_hard_loss"] + weight_soft_loss = input.extra_benchmark_config["weight_soft_loss"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + + torch_jsd_loss = TorchJSDLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + liger_jsd_loss = LigerJSDLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + + _tensor = torch.rand(BT, H // 2, device=device, dtype=dtype) + student_input1 = _tensor.detach().clone().requires_grad_(True) + student_input2 = _tensor.detach().clone().requires_grad_(True) + + teacher_input = torch.rand(BT, H, device=device, dtype=dtype) + + target = torch.randint(0, V, (BT,), device=device, dtype=torch.long) + + def fwd(): + if provider == "liger": + return liger_jsd_loss(student_input1, teacher_input, target) + elif provider == "torch": + return torch_jsd_loss(student_input2, teacher_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +def bench_speed_jsd_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + weight_hard_loss = input.extra_benchmark_config["weight_hard_loss"] + weight_soft_loss = input.extra_benchmark_config["weight_soft_loss"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + torch_jsd_loss = TorchJSDLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + liger_jsd_loss = LigerJSDLoss( + H=H, + V=V, + dtype=dtype, + ignore_index=ignore_index, + bias=bias, + weight_hard_loss=weight_hard_loss, + weight_soft_loss=weight_soft_loss, + ).to(device) + + _tensor = torch.rand(BT, H // 2, device=device, dtype=dtype) + student_input1 = _tensor.detach().clone().requires_grad_(True) + student_input2 = _tensor.detach().clone().requires_grad_(True) + + teacher_input = torch.rand(BT, H, device=device, dtype=dtype) + + target = torch.randint(0, V, (BT,), device=device, dtype=torch.long) + + def fwd(): + if provider == "liger": + return liger_jsd_loss(student_input1, teacher_input, target) + elif provider == "torch": + return torch_jsd_loss(student_input2, teacher_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[student_input1, student_input2], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + gpu_memory_gbs = get_total_gpu_memory() + # We know that the full test will require 69GBs for vocab size 2^13 and 39GBs for vocab size 2^12 on torch + if gpu_memory_gbs >= 69: + x_max = 13 + elif gpu_memory_gbs >= 39: + x_max = 12 + else: + x_max = 11 + + common_configs = { + "kernel_name": "distill_jsd_loss", + "x_name": "BT", + "x_label": "B x T", + "x_values": [2**i for i in range(10, x_max + 1)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "H": 4096, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + "bias": False, + "weight_hard_loss": 0.5, + "weight_soft_loss": 0.5, + "ignore_index": -100, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_jsd_loss, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + + run_benchmarks( + bench_test_fn=bench_memory_jsd_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_dpo_loss.py b/benchmark/scripts/benchmark_dpo_loss.py new file mode 100755 index 0000000..228a228 --- /dev/null +++ b/benchmark/scripts/benchmark_dpo_loss.py @@ -0,0 +1,179 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +def bench_memory_dpo_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_dpo_loss import LigerLMHeadDPO + from test.chunked_loss.test_dpo_loss import TorchLMHeadDPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + beta = input.extra_benchmark_config["beta"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + + # Instantiate once and retrieve the first output only + torch_dpo_loss = TorchLMHeadDPO(H=H, V=V, dtype=dtype, beta=beta, ignore_index=ignore_index, bias=bias).to(device) + liger_dpo_loss = LigerLMHeadDPO(H=H, V=V, dtype=dtype, beta=beta, ignore_index=ignore_index, bias=bias).to(device) + torch_fwd = lambda x, ref_x, target: torch_dpo_loss(x, ref_x, target)[0] + liger_fwd = lambda x, ref_x, target: liger_dpo_loss(x, ref_x, target)[0] + + # Input shape: [B, T, H] + _input = torch.randn(B, T, H, device=device, dtype=dtype) + ref_input = torch.randn(B, T, H, device=device, dtype=dtype, requires_grad=False) + # Target shape: [B, T] + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + # Add ignore_index tokens to simulate padding + num_elements_to_assign = torch.randint(1, B * T // 2, (1,)).item() + indices_to_assign = torch.randperm(B * T)[:num_elements_to_assign] + target.view(-1)[indices_to_assign] = ignore_index + + def fwd(): + if provider == "liger": + return liger_fwd(_input, ref_input, target) + elif provider == "huggingface": + return torch_fwd(_input, ref_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +def bench_speed_dpo_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_dpo_loss import LigerLMHeadDPO + from test.chunked_loss.test_dpo_loss import TorchLMHeadDPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + beta = input.extra_benchmark_config["beta"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + # Instantiate once and retrieve the first output only + torch_dpo_loss = TorchLMHeadDPO(H=H, V=V, dtype=dtype, beta=beta, ignore_index=ignore_index, bias=bias).to(device) + liger_dpo_loss = LigerLMHeadDPO(H=H, V=V, dtype=dtype, beta=beta, ignore_index=ignore_index, bias=bias).to(device) + torch_fwd = lambda x, ref_x, target: torch_dpo_loss(x, ref_x, target)[0] + liger_fwd = lambda x, ref_x, target: liger_dpo_loss(x, ref_x, target)[0] + + # Input shape: [B, T, H] + _input = torch.randn(B, T, H, device=device, dtype=dtype) + ref_input = torch.randn(B, T, H, device=device, dtype=dtype, requires_grad=False) + # Target shape: [B, T] + target = torch.randint(V, (B, T), device=device, dtype=torch.long) + + # Add ignore_index tokens + num_elements_to_assign = torch.randint(1, B * T // 2, (1,)).item() + indices_to_assign = torch.randperm(B * T)[:num_elements_to_assign] + target.view(-1)[indices_to_assign] = ignore_index + + def fwd(): + if provider == "liger": + return liger_fwd(_input, ref_input, target) + elif provider == "huggingface": + return torch_fwd(_input, ref_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "dpo_loss", + "x_name": "B", + "x_label": "Batch Size (B)", + "x_values": [2**i for i in range(1, 6)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "T": 512, + "H": 1024, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + "bias": True, + "beta": 0.1, + "ignore_index": 42, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_dpo_loss, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + + run_benchmarks( + bench_test_fn=bench_memory_dpo_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_dyt.py b/benchmark/scripts/benchmark_dyt.py new file mode 100755 index 0000000..2c51290 --- /dev/null +++ b/benchmark/scripts/benchmark_dyt.py @@ -0,0 +1,96 @@ +import os +import sys + +import torch + +from benchmark_model_configs import compute_hidden_size_sweep_config +from benchmark_model_configs import estimate_kernel_peak_memory +from benchmark_model_configs import get_benchmark_model_config +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import parse_benchmark_script_args +from utils import run_benchmarks +from utils import run_memory_benchmark +from utils import run_speed_benchmark + +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +def _setup_dyt(input: SingleBenchmarkRunInput): + """Create input tensor and DyT layer from benchmark config.""" + from test.transformers.test_dyt import LigerDyT + from test.transformers.test_dyt import TorchDyT + + cfg = input.extra_benchmark_config + hidden_size = input.x + x = torch.randn(cfg["BT"], hidden_size, device=device, dtype=cfg["dtype"], requires_grad=True) + if input.kernel_provider == "liger": + layer = LigerDyT(hidden_size=hidden_size, beta=cfg["beta"]).to(device) + elif input.kernel_provider == "torch": + layer = TorchDyT(hidden_size=hidden_size, beta=cfg["beta"]).to(device) + elif input.kernel_provider == "torch_compile": + layer = torch.compile(TorchDyT(hidden_size=hidden_size, beta=cfg["beta"]).to(device)) + else: + raise ValueError(f"Invalid provider: {input.kernel_provider} for DyT") + return x, layer + + +def bench_speed_dyt(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_dyt(input) + return run_speed_benchmark(lambda: layer(x), input.kernel_operation_mode, [x]) + + +def bench_memory_dyt(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_dyt(input) + return run_memory_benchmark(lambda: layer(x), input.kernel_operation_mode) + + +BT = 4096 + +if __name__ == "__main__": + args = parse_benchmark_script_args() + model = get_benchmark_model_config(args.model) + + for beta in [False, True]: + + def _probe(): + probe_input = SingleBenchmarkRunInput( + x=model.hidden_size, + kernel_provider="torch", + extra_benchmark_config={"BT": BT, "dtype": model.dtype, "beta": beta}, + ) + x, layer = _setup_dyt(probe_input) + return layer(x) + + peak_bytes = estimate_kernel_peak_memory(probe_fn=_probe) + sweep_config = compute_hidden_size_sweep_config(model, peak_bytes, bt=BT) + x_values = [1024 * i for i in range(1, 17) if 1024 * i <= sweep_config.max_hidden_size] or [model.hidden_size] + + common_configs = { + "kernel_name": f"dyt_beta={beta}", + "x_name": "hidden_size", + "x_label": "hidden_size", + "x_values": x_values, + "kernel_providers": ["liger", "torch", "torch_compile"], + "extra_benchmark_configs": [{"BT": sweep_config.bt, "dtype": model.dtype, "beta": beta}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_dyt, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_dyt, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_embedding.py b/benchmark/scripts/benchmark_embedding.py new file mode 100755 index 0000000..2bd0c60 --- /dev/null +++ b/benchmark/scripts/benchmark_embedding.py @@ -0,0 +1,134 @@ +import torch +import triton + +from torch.nn import Embedding +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.experimental.embedding import LigerEmbedding +from liger_kernel.utils import infer_device + +device = infer_device() + +# NOTE: For torch compile, we will just use default inductor settings. No further customization +# is needed. + + +def bench_speed_embedding(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + V = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + B = input.extra_benchmark_config["B"] + T = input.extra_benchmark_config["T"] + D = input.extra_benchmark_config["D"] + dtype = input.extra_benchmark_config["dtype"] + + torch_emb = Embedding(V, D).to(device).to(dtype) + liger_emb = LigerEmbedding(V, D).to(device).to(dtype) + torch_compile_emb = torch.compile(torch_emb) + + input_ids = torch.randint(0, V, (B, T), device=device) + + def fwd(): + if provider == "liger": + return liger_emb(input_ids) + elif provider == "torch_compile": + return torch_compile_emb(input_ids) + else: + return torch_emb(input_ids) + + def full(): + output = fwd() + output.backward(torch.randn_like(output)) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, quantiles=QUANTILES, rep=100) + elif mode == "backward": + output = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: output.backward(torch.randn_like(output), retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[input_ids], + rep=100, + ) + elif mode == "full": + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, rep=100) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_embedding(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + V = input.x + provider = input.kernel_provider + + B = input.extra_benchmark_config["B"] + T = input.extra_benchmark_config["T"] + D = input.extra_benchmark_config["D"] + dtype = input.extra_benchmark_config["dtype"] + + torch_emb = Embedding(V, D).to(device).to(dtype) + liger_emb = LigerEmbedding(V, D).to(device).to(dtype) + torch_compile_emb = torch.compile(torch_emb) + + input_ids = torch.randint(0, V, (B, T), device=device) + + def fwd(): + if provider == "liger": + return liger_emb(input_ids) + elif provider == "torch_compile": + return torch_compile_emb(input_ids) + else: + return torch_emb(input_ids) + + def full(): + output = fwd() + output.backward(torch.randn_like(output)) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "embedding", + "x_name": "V", + "x_label": "embedding dimension", + "x_values": [2**i for i in range(10, 18)], + "kernel_providers": ["liger", "huggingface", "torch_compile"], + "extra_benchmark_configs": [ + # BERT + {"B": 32, "T": 512, "D": 768, "dtype": torch.float32}, + # Llama + {"B": 8, "T": 2048, "D": 4096, "dtype": torch.float32}, + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_embedding, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_embedding, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_fused_add_rms_norm.py b/benchmark/scripts/benchmark_fused_add_rms_norm.py new file mode 100755 index 0000000..935871e --- /dev/null +++ b/benchmark/scripts/benchmark_fused_add_rms_norm.py @@ -0,0 +1,201 @@ +import torch +import torch.nn as nn +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.fused_add_rms_norm import LigerFusedAddRMSNorm +from liger_kernel.transformers.rms_norm import LigerRMSNorm +from liger_kernel.utils import infer_device + +device = infer_device() + + +class NaiveAddRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Naive implementation of the add residual rms norm. + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states, residual): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + residual = residual.to(torch.float32) + hidden_states = hidden_states + residual + residual = hidden_states + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype), residual.to(input_dtype) + + +class AddLigerRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + AddLigerRMSNorm is equivalent to NaiveAddRMSNorm class above, but uses the LigerRMSNorm kernel. + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + self.rms_norm = LigerRMSNorm(hidden_size, eps, in_place=False) + + def forward(self, hidden_states, residual): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + residual = residual.to(torch.float32) + hidden_states = hidden_states + residual + residual = hidden_states + hidden_states = self.rms_norm(hidden_states) + return self.weight * hidden_states.to(input_dtype), residual.to(input_dtype) + + +def bench_speed_fused_residual_rms_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + + # Fused Add RMS Norm + fused_add_rms_norm = LigerFusedAddRMSNorm(hidden_size=N, eps=eps).to(device) + # Naive implementation + naive_rms_norm = NaiveAddRMSNorm(hidden_size=N, eps=eps).to(device) + # LigerRMSNorm without fused residual addition + liger_rms_norm = AddLigerRMSNorm(hidden_size=N, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + r = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + ds = torch.randn_like(r) + x.requires_grad_(True) + r.requires_grad_(True) + # utility functions + + def y_fwd(): + if provider == "liger_fused_add_rms_norm": + return fused_add_rms_norm(x, r) + + if provider == "huggingface": + return naive_rms_norm(x, r) + + if provider == "liger_rms_norm": + return liger_rms_norm(x, r) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + y_fwd, + grad_to_none=[x, r], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "backward": + y, s = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: torch.autograd.backward((y, s), (dy, ds), retain_graph=True), + grad_to_none=[x, r], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y, s = y_fwd() + torch.autograd.backward((y, s), (dy, ds)) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[x, r], + rep=500, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_fused_residual_rms_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + + fused_add_rms_norm = LigerFusedAddRMSNorm(hidden_size=N, eps=eps).to(device) + naive_rms_norm = NaiveAddRMSNorm(hidden_size=N, eps=eps).to(device) + liger_rms_norm = AddLigerRMSNorm(hidden_size=N, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + r = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + ds = torch.randn_like(r) + x.requires_grad_(True) + r.requires_grad_(True) + + # utility functions + def y_fwd(): + if provider == "liger_fused_add_rms_norm": + return fused_add_rms_norm(x, r) + if provider == "huggingface": + return naive_rms_norm(x, r) + if provider == "liger_rms_norm": + return liger_rms_norm(x, r) + + def full(): + y, s = y_fwd() + torch.autograd.backward((y, s), (dy, ds)) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_add_rms_norm", + "x_name": "H", + "x_label": "hidden size", + "x_values": [2**i for i in range(10, 16)], + "kernel_providers": ["liger_fused_add_rms_norm", "huggingface", "liger_rms_norm"], + "extra_benchmark_configs": [{"M": 2048, "dtype": torch.float32, "eps": 1e-6}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_residual_rms_norm, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_residual_rms_norm, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_fused_linear_cross_entropy.py b/benchmark/scripts/benchmark_fused_linear_cross_entropy.py new file mode 100755 index 0000000..4d36a66 --- /dev/null +++ b/benchmark/scripts/benchmark_fused_linear_cross_entropy.py @@ -0,0 +1,184 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.fused_linear_cross_entropy import LigerFusedLinearCrossEntropyLoss +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchLMHeadCE(torch.nn.Module): + """Ground truth implementation of the linear fused with torch based cross entropy loss. + + :param H: hidden size + :param V: vocab size + :param ignore_index: index to ignore + :param reduction: reduction method + """ + + def __init__(self, H: int, V: int, dtype: torch.dtype, ignore_index: int = -100): + super().__init__() + self.lin = torch.nn.Linear(in_features=H, out_features=V, bias=False, dtype=dtype) + self.ce_loss = torch.nn.CrossEntropyLoss(ignore_index=ignore_index, reduction="mean") + + def forward(self, x, y): + logits = self.lin(x) + return self.ce_loss(logits, y) + + +class LigerLMHeadCE(torch.nn.Module): + def __init__(self, H: int, V: int, dtype: torch.dtype, ignore_index: int = -100, accum_dtype=None): + super().__init__() + self.lin = torch.nn.Linear(in_features=H, out_features=V, bias=False, dtype=dtype) + self.ce_loss = LigerFusedLinearCrossEntropyLoss( + ignore_index=ignore_index, reduction="mean", accum_dtype=accum_dtype + ) + + def forward(self, x, y): + return self.ce_loss(self.lin.weight, x, y) + + +############################################################################# +# Test the memory consumption of the linear fused cross entropy loss +############################################################################# + + +def bench_memory_fused_linear_cross_entropy( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + + lm_head_ce = None + if provider == "liger": + lm_head_ce = LigerLMHeadCE(H=H, V=V, dtype=dtype).to(device) + elif provider == "liger-fp32-accum": + lm_head_ce = LigerLMHeadCE(H=H, V=V, dtype=dtype, accum_dtype=torch.float32).to(device) + else: + lm_head_ce = TorchLMHeadCE(H=H, V=V, dtype=dtype).to(device) + + _input = torch.randn(BT, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (BT, 1), dtype=torch.long, device=device).squeeze(1) + + def fwd(): + return lm_head_ce(_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +# ############################################################################# +# # Test the speed of the fused linear cross entropy loss +# ############################################################################# + + +def bench_speed_fused_linear_cross_entropy( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + lm_head_ce = None + if provider == "liger": + lm_head_ce = LigerLMHeadCE(H=H, V=V, dtype=dtype).to(device) + elif provider == "liger-fp32-accum": + lm_head_ce = LigerLMHeadCE(H=H, V=V, dtype=dtype, accum_dtype=torch.float32).to(device) + else: + lm_head_ce = TorchLMHeadCE(H=H, V=V, dtype=dtype).to(device) + + _input = torch.randn(BT, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (BT, 1), dtype=torch.long, device=device).squeeze(1) + + def fwd(): + return lm_head_ce(_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "no-grad-forward": + with torch.no_grad(): + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_linear_cross_entropy", + "x_name": "BT", + "x_label": "B x T", + "x_values": [2**i for i in range(12, 16)], + "kernel_providers": ["liger", "liger-fp32-accum", "huggingface"], + "extra_benchmark_configs": [{"H": 4096, "V": 128256, "mode": "forward", "dtype": torch.bfloat16}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_cross_entropy, + kernel_operation_modes=["forward", "backward", "full", "no-grad-forward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_cross_entropy, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_fused_linear_jsd.py b/benchmark/scripts/benchmark_fused_linear_jsd.py new file mode 100755 index 0000000..ac62863 --- /dev/null +++ b/benchmark/scripts/benchmark_fused_linear_jsd.py @@ -0,0 +1,260 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.fused_linear_jsd import LigerFusedLinearJSD +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchJSD(torch.nn.Module): + def __init__( + self, + beta: float = 0.5, + ignore_index: int = -100, + dtype: torch.dtype = torch.float, + ): + super(TorchJSD, self).__init__() + self.kl = torch.nn.KLDivLoss(reduction="none", log_target=True) + self.beta = beta + self.ignore_index = ignore_index + self.dtype = dtype + + def forward( + self, + log_q: torch.Tensor, # input + log_p: torch.Tensor, # target + label=None, + ): + log_p, log_q = log_p.to(torch.float), log_q.to(torch.float) + log_p, log_q = log_p.view(-1, log_p.size(-1)), log_q.view(-1, log_q.size(-1)) + m = torch.lerp(torch.exp(log_q), torch.exp(log_p), self.beta) + loss = self.beta * self.kl(torch.log(m), log_p).sum(dim=-1) + (1 - self.beta) * self.kl( + torch.log(m), log_q + ).sum(dim=-1) + + if label is not None: + loss = torch.where(label != self.ignore_index, loss, 0.0) + n_non_ignore = (label != self.ignore_index).sum().item() + if n_non_ignore == 0: + loss = 0.0 + else: + loss = (loss / n_non_ignore).sum() + else: + loss = (loss / log_q.shape[0]).sum() + return loss.to(self.dtype) + + +class TorchLMHeadJSD(torch.nn.Module): + """Ground truth implementation of the linear fused with torch based jsd loss. + + :param H: hidden size + :param V: vocab size + :param temperature: softmax temperature + :param beta: jsd beta + """ + + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + device: torch.device, + beta: float = 0.5, + ignore_index: int = -100, + temperature: float = 1.0, + ): + super().__init__() + self.student_lin = torch.nn.Linear(in_features=H, out_features=V, bias=False, dtype=dtype, device=device) + self.teacher_lin = torch.nn.Linear(in_features=H, out_features=V, bias=False, dtype=dtype, device=device) + self.jsd = TorchJSD(beta=beta, ignore_index=ignore_index, dtype=dtype) + self.temperature = temperature + + def forward(self, student_input, teacher_input, label=None): + student_logits = self.student_lin(student_input) + teacher_logits = self.teacher_lin(teacher_input) + student_prob = torch.log_softmax(student_logits / self.temperature, dim=-1) + teacher_prob = torch.log_softmax(teacher_logits / self.temperature, dim=-1) + + return self.jsd(student_prob, teacher_prob, label) + + +class LigerLMHeadJSD(torch.nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + device: torch.device, + beta: float = 0.5, + ignore_index: int = -100, + temperature: float = 1.0, + ): + super().__init__() + self.student_lin = torch.nn.Linear(in_features=H, out_features=V, bias=False, dtype=dtype, device=device) + self.teacher_lin = torch.nn.Linear(in_features=H, out_features=V, bias=False, dtype=dtype, device=device) + self.fused_jsd = LigerFusedLinearJSD(jsd_beta=beta, ignore_index=ignore_index, temperature=temperature) + + def forward(self, student_input, teacher_input, label=None): + return self.fused_jsd( + student_input, + self.student_lin.weight, + teacher_input, + self.teacher_lin.weight, + label, + ) + + +############################################################################# +# Test the memory consumption of the fused linear JSD +############################################################################# + + +def bench_memory_fused_linear_jsd( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + + torch_lm_head_jsd = TorchLMHeadJSD(H=H, V=V, dtype=dtype, device=device).to(device) + liger_lm_head_jsd = LigerLMHeadJSD(H=H, V=V, dtype=dtype, device=device).to(device) + + # init the linear in all FusedLinearJSDs with the same weights + torch_lm_head_jsd.student_lin.weight.data = liger_lm_head_jsd.student_lin.weight.data = torch.rand( + V, H, device=device, dtype=dtype + ) + torch_lm_head_jsd.teacher_lin.weight.data = liger_lm_head_jsd.teacher_lin.weight.data = torch.rand( + V, H, device=device, dtype=dtype + ) + + student_input = torch.rand(BT, H, requires_grad=True, dtype=dtype, device=device) + teacher_input = torch.rand(BT, H, dtype=dtype, device=device) + + def fwd(): + if provider == "liger": + return liger_lm_head_jsd(student_input, teacher_input) + elif provider == "torch": + return torch_lm_head_jsd(student_input, teacher_input) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +# ############################################################################# +# # Test the speed of the fused linear JSD +# ############################################################################# + + +def bench_speed_fused_linear_jsd( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + BT = input.x + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + mode = input.kernel_operation_mode + + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + + torch_lm_head_jsd = TorchLMHeadJSD(H=H, V=V, dtype=dtype, device=device).to(device) + liger_lm_head_jsd = LigerLMHeadJSD(H=H, V=V, dtype=dtype, device=device).to(device) + + # init the linear in all FusedLinearJSDs with the same weights + torch_lm_head_jsd.student_lin.weight.data = liger_lm_head_jsd.student_lin.weight.data = torch.rand( + V, H, device=device, dtype=dtype + ) + torch_lm_head_jsd.teacher_lin.weight.data = liger_lm_head_jsd.teacher_lin.weight.data = torch.rand( + V, H, device=device, dtype=dtype + ) + + student_input = torch.rand(BT, H, requires_grad=True, dtype=dtype, device=device) + teacher_input = torch.rand(BT, H, dtype=dtype, device=device) + + def fwd(): + if provider == "liger": + return liger_lm_head_jsd(student_input, teacher_input) + elif provider == "torch": + return torch_lm_head_jsd(student_input, teacher_input) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[ + student_input, + torch_lm_head_jsd.student_lin.weight, + torch_lm_head_jsd.teacher_lin.weight, + ], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_linear_jsd", + "x_name": "BT", + "x_label": "B x T", + "x_values": [2**i for i in range(10, 14)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [{"H": 4096, "V": 128256, "mode": "forward", "dtype": torch.bfloat16}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_jsd, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_jsd, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_fused_neighborhood_attention.py b/benchmark/scripts/benchmark_fused_neighborhood_attention.py new file mode 100755 index 0000000..515d65c --- /dev/null +++ b/benchmark/scripts/benchmark_fused_neighborhood_attention.py @@ -0,0 +1,367 @@ +import math + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.fused_neighborhood_attention import LigerFusedNeighborhoodAttention +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchNeighborhoodAttention(torch.nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + kernel_size: int = 7, + dilation: int = 1, + bias: bool = True, + dropout: float = 0.0, + scale: float = None, + ): + super().__init__() + + if hidden_size % num_heads != 0: + raise ValueError(f"hidden_size ({hidden_size}) must be divisible by num_heads ({num_heads})") + + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.kernel_size = kernel_size + self.dilation = dilation + self.scale = scale if scale is not None else 1.0 / math.sqrt(self.head_dim) + + self.q_proj = torch.nn.Linear(hidden_size, hidden_size, bias=bias) + self.k_proj = torch.nn.Linear(hidden_size, hidden_size, bias=bias) + self.v_proj = torch.nn.Linear(hidden_size, hidden_size, bias=bias) + self.out_proj = torch.nn.Linear(hidden_size, hidden_size, bias=bias) + + if dropout > 0.0: + self.dropout = torch.nn.Dropout(dropout) + else: + self.dropout = None + + def _create_neighborhood_mask(self, seq_len: int, device: torch.device) -> torch.Tensor: + mask = torch.zeros(seq_len, seq_len, device=device, dtype=torch.bool) + half_kernel = self.kernel_size // 2 + + for i in range(seq_len): + start = max(0, i - half_kernel * self.dilation) + end = min(seq_len, i + half_kernel * self.dilation + 1) + + for j in range(start, end): + if self.dilation == 1 or (j - i) % self.dilation == 0: + mask[i, j] = True + + return mask + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, hidden_size = hidden_states.shape + + query = self.q_proj(hidden_states) + key = self.k_proj(hidden_states) + value = self.v_proj(hidden_states) + + query = query.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + key = key.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + value = value.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + + scores = torch.matmul(query, key.transpose(-2, -1)) * self.scale + + mask = self._create_neighborhood_mask(seq_len, hidden_states.device) + scores = scores.masked_fill(~mask, float("-inf")) + + attn_weights = torch.softmax(scores, dim=-1) + + if self.dropout is not None: + attn_weights = self.dropout(attn_weights) + + attn_output = torch.matmul(attn_weights, value) + + attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, seq_len, hidden_size) + + output = self.out_proj(attn_output) + + return output + + +def bench_speed_fused_neighborhood_attention(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + seq_len = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + batch_size = extra_benchmark_config["batch_size"] + hidden_size = extra_benchmark_config["hidden_size"] + num_heads = extra_benchmark_config["num_heads"] + kernel_size = extra_benchmark_config["kernel_size"] + dilation = extra_benchmark_config["dilation"] + bias = extra_benchmark_config["bias"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (batch_size, seq_len, hidden_size) + + liger_attn = ( + LigerFusedNeighborhoodAttention( + hidden_size=hidden_size, + num_heads=num_heads, + kernel_size=kernel_size, + dilation=dilation, + bias=bias, + dropout=0.0, + ) + .to(device) + .to(dtype) + ) + + torch_attn = ( + TorchNeighborhoodAttention( + hidden_size=hidden_size, + num_heads=num_heads, + kernel_size=kernel_size, + dilation=dilation, + bias=bias, + dropout=0.0, + ) + .to(device) + .to(dtype) + ) + + with torch.no_grad(): + torch_attn.q_proj.weight.copy_(liger_attn.q_proj.weight) + torch_attn.k_proj.weight.copy_(liger_attn.k_proj.weight) + torch_attn.v_proj.weight.copy_(liger_attn.v_proj.weight) + torch_attn.out_proj.weight.copy_(liger_attn.out_proj.weight) + + if bias: + torch_attn.q_proj.bias.copy_(liger_attn.q_proj.bias) + torch_attn.k_proj.bias.copy_(liger_attn.k_proj.bias) + torch_attn.v_proj.bias.copy_(liger_attn.v_proj.bias) + torch_attn.out_proj.bias.copy_(liger_attn.out_proj.bias) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def fwd(): + if provider == "liger": + return liger_attn(x) + elif provider == "torch": + return torch_attn(x) + + print(f"Starting Warmup for input size: {x_shape}") + _ = fwd() + if mode in ("backward", "full"): + y = _ + y.backward(dy, retain_graph=True) + print("Done Warmup") + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, grad_to_none=[x], rep=100, quantiles=QUANTILES) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + grad_to_none=[x], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, grad_to_none=[x], rep=100, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_fused_neighborhood_attention(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + seq_len = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + batch_size = extra_benchmark_config["batch_size"] + hidden_size = extra_benchmark_config["hidden_size"] + num_heads = extra_benchmark_config["num_heads"] + kernel_size = extra_benchmark_config["kernel_size"] + dilation = extra_benchmark_config["dilation"] + bias = extra_benchmark_config["bias"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (batch_size, seq_len, hidden_size) + + liger_attn = ( + LigerFusedNeighborhoodAttention( + hidden_size=hidden_size, + num_heads=num_heads, + kernel_size=kernel_size, + dilation=dilation, + bias=bias, + dropout=0.0, + ) + .to(device) + .to(dtype) + ) + + torch_attn = ( + TorchNeighborhoodAttention( + hidden_size=hidden_size, + num_heads=num_heads, + kernel_size=kernel_size, + dilation=dilation, + bias=bias, + dropout=0.0, + ) + .to(device) + .to(dtype) + ) + + with torch.no_grad(): + torch_attn.q_proj.weight.copy_(liger_attn.q_proj.weight) + torch_attn.k_proj.weight.copy_(liger_attn.k_proj.weight) + torch_attn.v_proj.weight.copy_(liger_attn.v_proj.weight) + torch_attn.out_proj.weight.copy_(liger_attn.out_proj.weight) + + if bias: + torch_attn.q_proj.bias.copy_(liger_attn.q_proj.bias) + torch_attn.k_proj.bias.copy_(liger_attn.k_proj.bias) + torch_attn.v_proj.bias.copy_(liger_attn.v_proj.bias) + torch_attn.out_proj.bias.copy_(liger_attn.out_proj.bias) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def fwd(): + if provider == "liger": + return liger_attn(x) + elif provider == "torch": + return torch_attn(x) + + def full(): + y = fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_neighborhood_attention", + "x_name": "seq_len", + "x_label": "sequence length", + "x_values": [2**i for i in range(6, 13)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "batch_size": 2, + "hidden_size": 512, + "num_heads": 8, + "kernel_size": 7, + "dilation": 1, + "bias": True, + "dtype": torch.float32, + }, + { + "batch_size": 4, + "hidden_size": 768, + "num_heads": 12, + "kernel_size": 7, + "dilation": 1, + "bias": True, + "dtype": torch.float32, + }, + { + "batch_size": 2, + "hidden_size": 1024, + "num_heads": 16, + "kernel_size": 9, + "dilation": 1, + "bias": True, + "dtype": torch.float32, + }, + { + "batch_size": 2, + "hidden_size": 512, + "num_heads": 8, + "kernel_size": 7, + "dilation": 2, + "bias": True, + "dtype": torch.float32, + }, + { + "batch_size": 2, + "hidden_size": 512, + "num_heads": 8, + "kernel_size": 7, + "dilation": 1, + "bias": True, + "dtype": torch.bfloat16, + }, + { + "batch_size": 4, + "hidden_size": 768, + "num_heads": 12, + "kernel_size": 7, + "dilation": 1, + "bias": True, + "dtype": torch.bfloat16, + }, + { + "batch_size": 2, + "hidden_size": 1024, + "num_heads": 16, + "kernel_size": 9, + "dilation": 1, + "bias": True, + "dtype": torch.bfloat16, + }, + { + "batch_size": 2, + "hidden_size": 512, + "num_heads": 8, + "kernel_size": 7, + "dilation": 2, + "bias": True, + "dtype": torch.bfloat16, + }, + ], + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_neighborhood_attention, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + + run_benchmarks( + bench_test_fn=bench_memory_fused_neighborhood_attention, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_geglu.py b/benchmark/scripts/benchmark_geglu.py new file mode 100755 index 0000000..d59564b --- /dev/null +++ b/benchmark/scripts/benchmark_geglu.py @@ -0,0 +1,115 @@ +import math + +import torch + +from benchmark_model_configs import compute_seq_len_sweep_config +from benchmark_model_configs import estimate_kernel_peak_memory +from benchmark_model_configs import get_benchmark_model_config +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import LlamaMLP +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import parse_benchmark_script_args +from utils import run_benchmarks +from utils import run_memory_benchmark +from utils import run_speed_benchmark + +from liger_kernel.transformers.geglu import LigerGEGLUMLP +from liger_kernel.utils import infer_device + +device = infer_device() + + +def _setup_geglu(input: SingleBenchmarkRunInput): + """Create input tensor and GEGLU layer from benchmark config.""" + cfg = input.extra_benchmark_config + llama_config = LlamaConfig( + hidden_size=cfg["hidden_size"], + intermediate_size=cfg["intermediate_size"], + hidden_act=cfg["hidden_act"], + ) + x = torch.randn( + cfg["bsz"], + input.x, + cfg["hidden_size"], + device=device, + dtype=cfg["dtype"], + requires_grad=True, + ) + if input.kernel_provider == "liger": + layer = LigerGEGLUMLP(config=llama_config).to(device).to(cfg["dtype"]) + elif input.kernel_provider == "huggingface": + layer = LlamaMLP(config=llama_config).to(device).to(cfg["dtype"]) + else: + raise ValueError(f"Invalid provider: {input.kernel_provider} for GEGLU") + return x, layer + + +def bench_speed_geglu(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_geglu(input) + return run_speed_benchmark(lambda: layer(x), input.kernel_operation_mode, [x]) + + +def bench_memory_geglu(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_geglu(input) + return run_memory_benchmark(lambda: layer(x), input.kernel_operation_mode) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + model = get_benchmark_model_config(args.model) + probe_seq_len = 1024 + + def _probe(): + probe_input = SingleBenchmarkRunInput( + x=probe_seq_len, + kernel_provider="huggingface", + extra_benchmark_config={ + "bsz": 1, + "hidden_size": model.hidden_size, + "intermediate_size": model.intermediate_size, + "hidden_act": "gelu_pytorch_tanh", + "dtype": model.dtype, + }, + ) + x, layer = _setup_geglu(probe_input) + return layer(x) + + peak_bytes = estimate_kernel_peak_memory(probe_fn=_probe) + kernel_bpt = peak_bytes // probe_seq_len + + config = compute_seq_len_sweep_config(model, kernel_bytes_per_token=kernel_bpt) + + common_configs = { + "kernel_name": "geglu", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, int(math.log2(config.seq_len)) + 1)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "bsz": config.batch_size, + "hidden_size": model.hidden_size, + "intermediate_size": model.intermediate_size, + "hidden_act": "gelu_pytorch_tanh", + "dtype": model.dtype, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_geglu, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_geglu, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_group_norm.py b/benchmark/scripts/benchmark_group_norm.py new file mode 100755 index 0000000..5a8bf37 --- /dev/null +++ b/benchmark/scripts/benchmark_group_norm.py @@ -0,0 +1,137 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.group_norm import LigerGroupNorm +from liger_kernel.utils import infer_device + +device = infer_device() + + +def bench_speed_group_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + C = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + H = extra_benchmark_config["H"] + channels_per_group = extra_benchmark_config["channels_per_group"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, C, H) + triton_ln = LigerGroupNorm(num_channels=C, num_groups=C // channels_per_group, eps=eps).to(device) + torch_ln = torch.nn.GroupNorm(num_groups=C // channels_per_group, num_channels=C, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def y_fwd(): + if provider == "liger": + return triton_ln(x) + if provider == "huggingface": + return torch_ln(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(y_fwd, quantiles=QUANTILES, grad_to_none=[x], rep=500) + elif mode == "backward": + y = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[x], + rep=500, + ) + elif mode == "full": + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, grad_to_none=[x], rep=500) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_group_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + C = input.x + provider = input.kernel_provider + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + H = extra_benchmark_config["H"] + channels_per_group = extra_benchmark_config["channels_per_group"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, C, H) + triton_ln = LigerGroupNorm(num_channels=C, num_groups=C // channels_per_group, eps=eps).to(device) + torch_ln = torch.nn.GroupNorm(num_groups=C // channels_per_group, num_channels=C, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def y_fwd(): + if provider == "liger": + return triton_ln(x) + if provider == "huggingface": + return torch_ln(x) + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "group_norm", + "x_name": "C", + "x_label": "num_channels", + "x_values": [2**i for i in range(5, 12)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "M": 128, + "H": 512, + "channels_per_group": 4, + "dtype": torch.float32, + "eps": 1e-6, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_group_norm, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_group_norm, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_grpo_loss.py b/benchmark/scripts/benchmark_grpo_loss.py new file mode 100755 index 0000000..497d869 --- /dev/null +++ b/benchmark/scripts/benchmark_grpo_loss.py @@ -0,0 +1,234 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +############################################################################# +# Test the memory consumption of the linear fused GRPO loss +############################################################################# + + +def bench_memory_fused_linear_grpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_grpo_loss import LigerLMHeadGRPO + from test.chunked_loss.test_grpo_loss import TorchLMHeadGRPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + importance_sampling_level = input.extra_benchmark_config["importance_sampling_level"] + provider = input.kernel_provider + + # Instantiate once and retrieve the first output only + torch_lm_head_grpo = TorchLMHeadGRPO(H=H, V=V, dtype=dtype, importance_sampling_level=importance_sampling_level).to( + device + ) + liger_lm_head_grpo = LigerLMHeadGRPO(H=H, V=V, dtype=dtype, importance_sampling_level=importance_sampling_level).to( + device + ) + + # Create inputs + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + selected_token_ids = torch.randint(0, V, (B, T), dtype=torch.long, device=device) + attention_mask = torch.ones(B, T, device=device) + advantages = torch.randn(B, dtype=dtype, device=device) + ref_input = torch.randn(B, T, H, dtype=dtype, device=device) + + torch_fwd = lambda: torch_lm_head_grpo(_input, selected_token_ids, attention_mask, advantages, ref_input=ref_input)[ + 0 + ] + liger_fwd = lambda: liger_lm_head_grpo(_input, selected_token_ids, attention_mask, advantages, ref_input=ref_input)[ + 0 + ] + + def fwd(): + if provider == "liger": + return liger_fwd() + elif provider == "torch": + return torch_fwd() + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +############################################################################# +# Test the speed of the fused linear GRPO loss +############################################################################# + + +def bench_speed_fused_linear_grpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_grpo_loss import LigerLMHeadGRPO + from test.chunked_loss.test_grpo_loss import TorchLMHeadGRPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + importance_sampling_level = input.extra_benchmark_config["importance_sampling_level"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + # Instantiate once and retrieve the first output only + torch_lm_head_grpo = TorchLMHeadGRPO(H=H, V=V, dtype=dtype, importance_sampling_level=importance_sampling_level).to( + device + ) + liger_lm_head_grpo = LigerLMHeadGRPO(H=H, V=V, dtype=dtype, importance_sampling_level=importance_sampling_level).to( + device + ) + + # Create inputs + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + selected_token_ids = torch.randint(0, V, (B, T), dtype=torch.long, device=device) + attention_mask = torch.ones(B, T, device=device) + advantages = torch.randn(B, dtype=dtype, device=device) + ref_input = torch.randn(B, T, H, dtype=dtype, device=device) + + torch_fwd = lambda: torch_lm_head_grpo(_input, selected_token_ids, attention_mask, advantages, ref_input=ref_input)[ + 0 + ] + liger_fwd = lambda: liger_lm_head_grpo(_input, selected_token_ids, attention_mask, advantages, ref_input=ref_input)[ + 0 + ] + + def fwd(): + if provider == "liger": + return liger_fwd() + elif provider == "torch": + return torch_fwd() + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + # Benchmark token-level importance sampling (original GRPO) + token_configs = { + "kernel_name": "fused_linear_grpo_loss_token", + "x_name": "B", + "x_label": "B", + "x_values": [2**i for i in range(1, 5)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "T": 1024, + "H": 4096, + "V": 128256, + "importance_sampling_level": "token", + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + # Benchmark sequence-level importance sampling (GSPO) + sequence_configs = { + "kernel_name": "fused_linear_grpo_loss_sequence", + "x_name": "B", + "x_label": "B", + "x_values": [2**i for i in range(1, 5)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "T": 1024, + "H": 4096, + "V": 128256, + "importance_sampling_level": "sequence", + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + # Run benchmarks for token-level (GRPO) + print("Benchmarking GRPO (token-level importance sampling)...") + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_grpo_loss, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **token_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_grpo_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **token_configs, + ) + + # Run benchmarks for sequence-level (GSPO) + print("Benchmarking GSPO (sequence-level importance sampling)...") + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_grpo_loss, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **sequence_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_grpo_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **sequence_configs, + ) diff --git a/benchmark/scripts/benchmark_jsd.py b/benchmark/scripts/benchmark_jsd.py new file mode 100755 index 0000000..16d71ea --- /dev/null +++ b/benchmark/scripts/benchmark_jsd.py @@ -0,0 +1,157 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.jsd import LigerJSD +from liger_kernel.utils import get_total_gpu_memory +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchJSD(torch.nn.Module): + def __init__( + self, + beta: float = 0.5, + ignore_index: int = -100, + dtype: torch.dtype = torch.float, + ): + super(TorchJSD, self).__init__() + self.kl = torch.nn.KLDivLoss(reduction="none", log_target=True) + self.beta = beta + self.ignore_index = ignore_index + self.dtype = dtype + + def forward( + self, + log_q: torch.Tensor, # input + log_p: torch.Tensor, # target + label=None, + ): + log_p, log_q = log_p.to(torch.float), log_q.to(torch.float) + log_p, log_q = log_p.view(-1, log_p.size(-1)), log_q.view(-1, log_q.size(-1)) + m = torch.lerp(torch.exp(log_q), torch.exp(log_p), self.beta) + loss = self.beta * self.kl(torch.log(m), log_p).sum(dim=-1) + (1 - self.beta) * self.kl( + torch.log(m), log_q + ).sum(dim=-1) + + if label is not None: + loss = torch.where(label != self.ignore_index, loss, 0.0) + n_non_ignore = (label != self.ignore_index).sum().item() + if n_non_ignore == 0: + loss = 0.0 + else: + loss = (loss / n_non_ignore).sum() + else: + loss = (loss / log_q.shape[0]).sum() + return loss.to(self.dtype) + + +def bench_speed_jsd(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + V = input.x + B, T = input.extra_benchmark_config["B"], input.extra_benchmark_config["T"] + torch_jsd = TorchJSD() + liger_jsd = LigerJSD() + + _input = torch.randn(B * T, V, requires_grad=True, device=device).log_softmax(dim=-1) + target = torch.randn(B * T, V, device=device).log_softmax(dim=-1) + + def fwd(): + if input.kernel_provider == "liger": + return liger_jsd(_input, target) + else: + return torch_jsd(_input, target) + + if input.kernel_operation_mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, quantiles=QUANTILES, rep=100) + elif input.kernel_operation_mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[_input], + rep=100, + ) + elif input.kernel_operation_mode == "full": + + def full(): + y = fwd() + y.backward(retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, rep=100) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_jsd(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + torch_jsd = TorchJSD() + liger_jsd = LigerJSD() + + V = input.x + B, T = input.extra_benchmark_config["B"], input.extra_benchmark_config["T"] + + _input = torch.randn(B * T, V, requires_grad=True, device=device).log_softmax(dim=-1) + target = torch.randn(B * T, V, device=device).log_softmax(dim=-1) + + def fwd(): + if input.kernel_provider == "liger": + return liger_jsd(_input, target) + else: + return torch_jsd(_input, target) + + def full(): + y = fwd() + y.backward(retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + gpu_memory_gbs = get_total_gpu_memory() + # We know that the full test will require 54GBs for vocab size 2^17 on torch + if gpu_memory_gbs >= 54: + x_max = 17 + else: + x_max = 16 + common_args = { + "kernel_name": "jsd", + "x_name": "V", + "x_label": "vocab size", + "x_values": [2**i for i in range(12, x_max + 1)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [{"B": 4, "T": 2048}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_memory_jsd, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_args, + ) + + run_benchmarks( + bench_test_fn=bench_speed_jsd, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_args, + ) diff --git a/benchmark/scripts/benchmark_kl_div.py b/benchmark/scripts/benchmark_kl_div.py new file mode 100755 index 0000000..09948c3 --- /dev/null +++ b/benchmark/scripts/benchmark_kl_div.py @@ -0,0 +1,117 @@ +import torch +import torch.nn as nn +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.kl_div import LigerKLDIVLoss +from liger_kernel.utils import infer_device + +device = infer_device() + +S, E = 12, 18 + + +def bench_speed_kldiv(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + reduction = "batchmean" + V = input.x + B, T = input.extra_benchmark_config["B"], input.extra_benchmark_config["T"] + torch_kl_div = nn.KLDivLoss(reduction=reduction) + liger_kl_div = LigerKLDIVLoss(reduction=reduction) + + _input = torch.randn(B * T, V, requires_grad=True, device=device).log_softmax(dim=-1) + target = torch.randn(B * T, V, device=device).softmax(dim=-1) + + def fwd(): + if input.kernel_provider == "liger": + return liger_kl_div(_input, target) + else: + return torch_kl_div(_input, target) + + if input.kernel_operation_mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, quantiles=QUANTILES, rep=100) + elif input.kernel_operation_mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[_input], + rep=100, + ) + elif input.kernel_operation_mode == "full": + + def full(): + y = fwd() + y.backward(retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, rep=100) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_kldiv(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + reduction = "batchmean" + torch_kl_div = nn.KLDivLoss(reduction=reduction) + liger_kl_div = LigerKLDIVLoss(reduction=reduction) + + V = input.x + B, T = input.extra_benchmark_config["B"], input.extra_benchmark_config["T"] + + _input = torch.randn(B * T, V, requires_grad=True, device=device).log_softmax(dim=-1) + target = torch.randn(B * T, V, device=device).softmax(dim=-1) + + def fwd(): + if input.kernel_provider == "liger": + return liger_kl_div(_input, target) + else: + return torch_kl_div(_input, target) + + def full(): + y = fwd() + y.backward(retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + common_args = { + "kernel_name": "kl_div", + "x_name": "V", + "x_label": "vocab size", + "x_values": [2**i for i in range(12, 18)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [{"B": 8, "T": 512}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_memory_kldiv, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_args, + ) + + run_benchmarks( + bench_test_fn=bench_speed_kldiv, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_args, + ) diff --git a/benchmark/scripts/benchmark_kto_loss.py b/benchmark/scripts/benchmark_kto_loss.py new file mode 100755 index 0000000..bbde1d5 --- /dev/null +++ b/benchmark/scripts/benchmark_kto_loss.py @@ -0,0 +1,314 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.chunked_loss import LigerFusedLinearKTOLoss +from liger_kernel.utils import infer_device + +device = infer_device() +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +class TorchLMHeadKTO(torch.nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + use_bias: bool = False, + use_ref_bias: bool = False, + ignore_index: int = -100, + beta: float = 0.1, + ): + from test.chunked_loss.test_kto_loss import HFKTOLoss + + super().__init__() + self.lin = torch.nn.Linear(in_features=H, out_features=V, bias=use_bias, dtype=dtype) + self.ref_lin = torch.nn.Linear(in_features=H, out_features=V, bias=use_ref_bias, dtype=dtype) + self.KTO_loss = HFKTOLoss( + ignore_index=ignore_index, + beta=beta, + use_ref_model=True, + ).get_batch_loss_metrics + + def forward(self, x, ref_x, y, preference_labels, kl=None): + return self.KTO_loss( + weight=self.lin.weight, + _input=x, + target=y, + bias=self.lin.bias, + ref_input=ref_x, + ref_weight=self.ref_lin.weight, + ref_bias=self.ref_lin.bias, + preference_labels=preference_labels, + kl=kl, + ) + + +class LigerLMHeadKTO(torch.nn.Module): + def __init__( + self, + H: int, + V: int, + dtype: torch.dtype, + use_bias: bool = False, + use_ref_bias: bool = False, + ignore_index: int = -100, + beta: float = 0.1, + ): + super().__init__() + self.lin = torch.nn.Linear(in_features=H, out_features=V, bias=use_bias, dtype=dtype) + self.ref_lin = torch.nn.Linear(in_features=H, out_features=V, bias=use_ref_bias, dtype=dtype) + self.KTO_loss = LigerFusedLinearKTOLoss( + ignore_index=ignore_index, + beta=beta, + use_ref_model=True, + ) + + def forward(self, x, ref_x, y, preference_labels, kl=None): + return self.KTO_loss( + _input=x, + lin_weight=self.lin.weight, + target=y, + preference_labels=preference_labels, + bias=self.lin.bias, + ref_input=ref_x, + ref_weight=self.ref_lin.weight, + ref_bias=self.ref_lin.bias, + kl=kl, + ) + + +def bench_memory_kto_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + beta = input.extra_benchmark_config["beta"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + + torch_kto_loss = TorchLMHeadKTO( + H=H, + V=V, + dtype=dtype, + use_bias=bias, + use_ref_bias=bias, + ignore_index=ignore_index, + beta=beta, + ).to(device) + + liger_kto_loss = LigerLMHeadKTO( + H=H, + V=V, + dtype=dtype, + use_bias=bias, + use_ref_bias=bias, + ignore_index=ignore_index, + beta=beta, + ).to(device) + + # Input shape: [B, T, H] + _input = torch.randn(B, T, H, device=device, dtype=dtype) + + # Target shape: [B, T] + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + # Preference labels shape: [B] + # Create binary preference labels (0 or 1) for each sequence in the batch + # Used to indicate preferred sequences (1) vs non-preferred sequences (0) + preference_labels = torch.randint(2, (B,), dtype=torch.bool, device=device) + + # Precomputed KL divergence between policy and reference distributions + kl = torch.randn(1, device=device, dtype=dtype) + + # Add ignore_index tokens to simulate padding + num_elements_to_assign = torch.randint(1, B * T // 2, (1,)).item() + indices_to_assign = torch.randperm(B * T)[:num_elements_to_assign] + target.view(-1)[indices_to_assign] = ignore_index + + # Add ref_x with the same shape as _input + ref_input = torch.randn(B, T, H, device=device, dtype=dtype) + + def fwd(): + if provider == "liger": + return liger_kto_loss( + x=_input, + ref_x=ref_input, + y=target, + preference_labels=preference_labels, + kl=kl, + )[0] + elif provider == "huggingface": + return torch_kto_loss( + x=_input, + ref_x=ref_input, + y=target, + preference_labels=preference_labels, + kl=kl, + )[0] + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +def bench_speed_kto_loss(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + bias = input.extra_benchmark_config["bias"] + beta = input.extra_benchmark_config["beta"] + ignore_index = input.extra_benchmark_config["ignore_index"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + torch_kto_loss = TorchLMHeadKTO( + H=H, + V=V, + dtype=dtype, + beta=beta, + ignore_index=ignore_index, + use_bias=bias, + ).to(device) + liger_kto_loss = LigerLMHeadKTO( + H=H, + V=V, + dtype=dtype, + beta=beta, + ignore_index=ignore_index, + use_bias=bias, + ).to(device) + + # Input shape: [B, T, H] + _input = torch.randn(B, T, H, device=device, dtype=dtype) + + # Target shape: [B, T] + target = torch.randint(V, (B, T), device=device, dtype=torch.long) + + # Preference labels shape: [B] + # Create binary preference labels (0 or 1) for each sequence in the batch + # Used to indicate preferred sequences (1) vs non-preferred sequences (0) + preference_labels = torch.randint(2, (B,), dtype=torch.bool, device=device) + + # Precomputed KL divergence between policy and reference distributions + kl = torch.randn(1, device=device, dtype=dtype) + + # Add ignore_index tokens + num_elements_to_assign = torch.randint(1, B * T // 2, (1,)).item() + indices_to_assign = torch.randperm(B * T)[:num_elements_to_assign] + target.view(-1)[indices_to_assign] = ignore_index + + # Add ref_x with the same shape as _input + ref_input = torch.randn(B, T, H, device=device, dtype=dtype) + + def fwd(): + if provider == "liger": + return liger_kto_loss( + x=_input, + ref_x=ref_input, + y=target, + preference_labels=preference_labels, + kl=kl, + )[0] + elif provider == "huggingface": + return torch_kto_loss( + x=_input, + ref_x=ref_input, + y=target, + preference_labels=preference_labels, + kl=kl, + )[0] + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "kto_loss", + "x_name": "B", + "x_label": "Batch Size (B)", + "x_values": [2**i for i in range(1, 6)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "T": 512, + "H": 1024, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + "bias": True, + "beta": 0.1, + "ignore_index": 42, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_kto_loss, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + + run_benchmarks( + bench_test_fn=bench_memory_kto_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_layer_norm.py b/benchmark/scripts/benchmark_layer_norm.py new file mode 100755 index 0000000..0addf78 --- /dev/null +++ b/benchmark/scripts/benchmark_layer_norm.py @@ -0,0 +1,125 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.layer_norm import LigerLayerNorm +from liger_kernel.utils import infer_device + +device = infer_device() + + +def bench_speed_layer_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + triton_ln = LigerLayerNorm(hidden_size=N).to(device) + torch_ln = torch.nn.LayerNorm(N, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def y_fwd(): + if provider == "liger": + return triton_ln(x) + if provider == "huggingface": + return torch_ln(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(y_fwd, quantiles=QUANTILES, grad_to_none=[x], rep=500) + elif mode == "backward": + y = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[x], + rep=500, + ) + elif mode == "full": + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, grad_to_none=[x], rep=500) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_layer_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + dtype = input.extra_benchmark_config["dtype"] + M = input.extra_benchmark_config["M"] + eps = input.extra_benchmark_config["eps"] + + x_shape = (M, N) + + triton_ln = LigerLayerNorm(hidden_size=N).to(device) + torch_ln = torch.nn.LayerNorm(N, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def y_fwd(): + if provider == "liger": + return triton_ln(x) + if provider == "huggingface": + return torch_ln(x) + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "layer_norm", + "x_name": "N", + "x_label": "hidden size", + "x_values": [2**i for i in range(10, 15)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [{"M": 4096, "dtype": torch.float32, "eps": 1e-6}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_layer_norm, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_layer_norm, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_llama4_rope.py b/benchmark/scripts/benchmark_llama4_rope.py new file mode 100755 index 0000000..47d0605 --- /dev/null +++ b/benchmark/scripts/benchmark_llama4_rope.py @@ -0,0 +1,245 @@ +import torch +import triton + +from transformers.models.llama4.configuration_llama4 import Llama4TextConfig +from transformers.models.llama4.modeling_llama4 import Llama4TextRotaryEmbedding +from transformers.models.llama4.modeling_llama4 import apply_rotary_emb +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.llama4_rope import liger_llama4_text_rotary_pos_emb +from liger_kernel.utils import infer_device +from liger_kernel.utils import transformers_version_dispatch + +device = infer_device() + + +def bench_speed_llama4_rope(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + num_q_heads = extra_benchmark_config["num_q_heads"] + num_kv_heads = extra_benchmark_config["num_kv_heads"] + dtype = extra_benchmark_config["dtype"] + + # x can be either hidden_size or seq_len + hidden_size = extra_benchmark_config["hidden_size"] if "hidden_size" in extra_benchmark_config else input.x + seq_len = extra_benchmark_config["seq_len"] if "seq_len" in extra_benchmark_config else input.x + + head_dim = hidden_size // num_q_heads + + # Create Llama4TextConfig for the rotary embedding + config = Llama4TextConfig( + hidden_size=hidden_size, + num_attention_heads=num_q_heads, + num_key_value_heads=num_kv_heads, + head_dim=head_dim, + max_position_embeddings=seq_len, + ) + + rotary_emb = transformers_version_dispatch( + "4.48.0", + Llama4TextRotaryEmbedding, + Llama4TextRotaryEmbedding, + before_kwargs={"config": config, "device": device}, + after_kwargs={"config": config, "device": device}, + ) + + q = torch.randn( + (1, seq_len, num_q_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ) + k = torch.randn( + (1, seq_len, num_kv_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ) + dq, dk = ( + torch.randn_like(q, device=device, dtype=dtype), + torch.randn_like(k, device=device), + ) + pos_ids = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0) + freqs_cis = rotary_emb(q, pos_ids) + + def fwd(): + if provider == "liger": + return liger_llama4_text_rotary_pos_emb(q, k, freqs_cis) + elif provider == "huggingface": + return apply_rotary_emb(q, k, freqs_cis) + else: + raise ValueError(f"Invalid provider: {provider} for Llama4 RoPE embedding") + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + elif mode == "backward": + q_out, k_out = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True, retain_graph=True), + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + q_out, k_out = fwd() + torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_llama4_rope(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + num_q_heads = extra_benchmark_config["num_q_heads"] + num_kv_heads = extra_benchmark_config["num_kv_heads"] + dtype = extra_benchmark_config["dtype"] + + # x can be either hidden_size or seq_len + hidden_size = extra_benchmark_config["hidden_size"] if "hidden_size" in extra_benchmark_config else input.x + seq_len = extra_benchmark_config["seq_len"] if "seq_len" in extra_benchmark_config else input.x + + head_dim = hidden_size // num_q_heads + + # Create Llama4TextConfig for the rotary embedding + config = Llama4TextConfig( + hidden_size=hidden_size, + num_attention_heads=num_q_heads, + num_key_value_heads=num_kv_heads, + head_dim=head_dim, + max_position_embeddings=seq_len, + ) + + rotary_emb = transformers_version_dispatch( + "4.48.0", + Llama4TextRotaryEmbedding, + Llama4TextRotaryEmbedding, + before_kwargs={"config": config, "device": device}, + after_kwargs={"config": config, "device": device}, + ) + + q = torch.randn( + (1, seq_len, num_q_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ) + k = torch.randn( + (1, seq_len, num_kv_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ) + dq, dk = ( + torch.randn_like(q, device=device, dtype=dtype), + torch.randn_like(k, device=device), + ) + pos_ids = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0) + freqs_cis = rotary_emb(q, pos_ids) + + def full(): + if provider == "liger": + q_out, k_out = liger_llama4_text_rotary_pos_emb(q, k, freqs_cis) + else: + q_out, k_out = apply_rotary_emb(q, k, freqs_cis) + torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory( + full, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs_varying_hidden_size = { + "kernel_name": "llama4_rope", + "x_name": "H", + "x_label": "hidden size", + "x_values": [32 * (2**i) for i in range(4, 10, 2)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "dtype": torch.bfloat16, + "seq_len": 2048, + "num_q_heads": 32, + "num_kv_heads": 8, + } + ], + "overwrite": args.overwrite, + } + run_benchmarks( + bench_test_fn=bench_speed_llama4_rope, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs_varying_hidden_size, + ) + run_benchmarks( + bench_test_fn=bench_memory_llama4_rope, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs_varying_hidden_size, + ) + + common_configs_varying_seq_len = { + "kernel_name": "llama4_rope", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, 15)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "dtype": torch.bfloat16, + "hidden_size": 8192, + "num_q_heads": 32, + "num_kv_heads": 8, + } + ], + "overwrite": args.overwrite, + } + run_benchmarks( + bench_test_fn=bench_speed_llama4_rope, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs_varying_seq_len, + ) + run_benchmarks( + bench_test_fn=bench_memory_llama4_rope, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs_varying_seq_len, + ) diff --git a/benchmark/scripts/benchmark_mhc.py b/benchmark/scripts/benchmark_mhc.py new file mode 100755 index 0000000..47cdd63 --- /dev/null +++ b/benchmark/scripts/benchmark_mhc.py @@ -0,0 +1,255 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.functional import liger_mhc_coeffs +from liger_kernel.transformers.functional import liger_mhc_post_res +from liger_kernel.transformers.functional import liger_mhc_pre +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +def bench_speed_mhc(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + from test.transformers.test_mhc import mhc_coeffs_ref + + T = input.x + B = input.extra_benchmark_config["B"] + HC = input.extra_benchmark_config["HC"] + C = input.extra_benchmark_config["C"] + sub_kernel = input.extra_benchmark_config["sub_kernel"] + tmax = input.extra_benchmark_config["tmax"] + rms_eps = input.extra_benchmark_config["rms_eps"] + pre_eps = input.extra_benchmark_config["pre_eps"] + sinkhorn_eps = input.extra_benchmark_config["sinkhorn_eps"] + post_mult = input.extra_benchmark_config["post_mult"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + coeffs_cfg = dict(tmax=tmax, rms_eps=rms_eps, pre_eps=pre_eps, sinkhorn_eps=sinkhorn_eps, post_mult=post_mult) + need_grad = mode in ("backward", "full") + + x = torch.randn(B, T, HC, C, device=device, dtype=torch.bfloat16, requires_grad=need_grad) + K, M = HC * C, HC * HC + 2 * HC + phi = (torch.randn(K, M, device=device, dtype=torch.bfloat16) * 0.02).requires_grad_(need_grad) + b_param = torch.zeros(M, device=device, dtype=torch.float32, requires_grad=need_grad) + alpha_pre = torch.tensor(1.0, device=device, dtype=torch.float32, requires_grad=need_grad) + alpha_post = torch.tensor(1.0, device=device, dtype=torch.float32, requires_grad=need_grad) + alpha_res = torch.tensor(1.0, device=device, dtype=torch.float32, requires_grad=need_grad) + + grad_to_none = [x, phi, b_param, alpha_pre, alpha_post, alpha_res] if need_grad else None + + if sub_kernel == "coeffs": + + def fwd(): + if provider == "liger": + return liger_mhc_coeffs(x, phi, b_param, alpha_pre, alpha_post, alpha_res, **coeffs_cfg) + return mhc_coeffs_ref(x, phi, b_param, alpha_pre, alpha_post, alpha_res, **coeffs_cfg) + + def fwd_loss(): + h_pre, h_post, h_res = fwd() + return h_pre.square().mean() + h_post.square().mean() + h_res.square().mean() + + elif sub_kernel == "pre": + with torch.no_grad(): + h_pre_c, _, _ = liger_mhc_coeffs( + x.detach(), + phi.detach(), + b_param.detach(), + alpha_pre.detach(), + alpha_post.detach(), + alpha_res.detach(), + **coeffs_cfg, + ) + h_pre_c.requires_grad_(need_grad) + grad_to_none = [x, h_pre_c] if need_grad else None + + def fwd(): + if provider == "liger": + return liger_mhc_pre(x, h_pre_c) + return (x.float() * h_pre_c.unsqueeze(-1)).sum(dim=-2) + + def fwd_loss(): + return fwd().square().mean() + + elif sub_kernel == "post_res": + with torch.no_grad(): + _, h_post_c, h_res_c = liger_mhc_coeffs( + x.detach(), + phi.detach(), + b_param.detach(), + alpha_pre.detach(), + alpha_post.detach(), + alpha_res.detach(), + **coeffs_cfg, + ) + h_post_c.requires_grad_(need_grad) + h_res_c.requires_grad_(need_grad) + f_out = torch.randn(B, T, C, device=device, dtype=torch.bfloat16, requires_grad=need_grad) + grad_to_none = [x, f_out, h_post_c, h_res_c] if need_grad else None + + def fwd(): + if provider == "liger": + return liger_mhc_post_res(x, f_out, h_post_c, h_res_c) + return torch.einsum("...oi,...ic->...oc", h_res_c, x.float()) + h_post_c.unsqueeze( + -1 + ) * f_out.float().unsqueeze(-2) + + def fwd_loss(): + return fwd().square().mean() + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, rep=100, quantiles=QUANTILES) + elif mode == "backward": + y = fwd_loss() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=grad_to_none, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd_loss() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, grad_to_none=grad_to_none, rep=100, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput(y_20=ms_20, y_50=ms_50, y_80=ms_80) + + +def bench_memory_mhc(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + from test.transformers.test_mhc import mhc_coeffs_ref + + T = input.x + B = input.extra_benchmark_config["B"] + HC = input.extra_benchmark_config["HC"] + C = input.extra_benchmark_config["C"] + sub_kernel = input.extra_benchmark_config["sub_kernel"] + tmax = input.extra_benchmark_config["tmax"] + rms_eps = input.extra_benchmark_config["rms_eps"] + pre_eps = input.extra_benchmark_config["pre_eps"] + sinkhorn_eps = input.extra_benchmark_config["sinkhorn_eps"] + post_mult = input.extra_benchmark_config["post_mult"] + provider = input.kernel_provider + + coeffs_cfg = dict(tmax=tmax, rms_eps=rms_eps, pre_eps=pre_eps, sinkhorn_eps=sinkhorn_eps, post_mult=post_mult) + + x = torch.randn(B, T, HC, C, device=device, dtype=torch.bfloat16, requires_grad=True) + K, M = HC * C, HC * HC + 2 * HC + phi = (torch.randn(K, M, device=device, dtype=torch.bfloat16) * 0.02).requires_grad_(True) + b_param = torch.zeros(M, device=device, dtype=torch.float32, requires_grad=True) + alpha_pre = torch.tensor(1.0, device=device, dtype=torch.float32, requires_grad=True) + alpha_post = torch.tensor(1.0, device=device, dtype=torch.float32, requires_grad=True) + alpha_res = torch.tensor(1.0, device=device, dtype=torch.float32, requires_grad=True) + + if sub_kernel == "coeffs": + + def full(): + if provider == "liger": + hp, hpo, hr = liger_mhc_coeffs(x, phi, b_param, alpha_pre, alpha_post, alpha_res, **coeffs_cfg) + else: + hp, hpo, hr = mhc_coeffs_ref(x, phi, b_param, alpha_pre, alpha_post, alpha_res, **coeffs_cfg) + (hp.square().mean() + hpo.square().mean() + hr.square().mean()).backward() + + elif sub_kernel == "pre": + with torch.no_grad(): + h_pre_c, _, _ = liger_mhc_coeffs( + x.detach(), + phi.detach(), + b_param.detach(), + alpha_pre.detach(), + alpha_post.detach(), + alpha_res.detach(), + **coeffs_cfg, + ) + h_pre_c.requires_grad_(True) + + def full(): + if provider == "liger": + out = liger_mhc_pre(x, h_pre_c) + else: + out = (x.float() * h_pre_c.unsqueeze(-1)).sum(dim=-2) + out.square().mean().backward() + + elif sub_kernel == "post_res": + with torch.no_grad(): + _, h_post_c, h_res_c = liger_mhc_coeffs( + x.detach(), + phi.detach(), + b_param.detach(), + alpha_pre.detach(), + alpha_post.detach(), + alpha_res.detach(), + **coeffs_cfg, + ) + h_post_c.requires_grad_(True) + h_res_c.requires_grad_(True) + f_out = torch.randn(B, T, C, device=device, dtype=torch.bfloat16, requires_grad=True) + + def full(): + if provider == "liger": + out = liger_mhc_post_res(x, f_out, h_post_c, h_res_c) + else: + out = torch.einsum("...oi,...ic->...oc", h_res_c, x.float()) + h_post_c.unsqueeze( + -1 + ) * f_out.float().unsqueeze(-2) + out.square().mean().backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput(y_20=mem_20, y_50=mem_50, y_80=mem_80) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + for sub_kernel in ["coeffs", "pre", "post_res"]: + common_configs = { + "kernel_name": f"mhc_{sub_kernel}", + "x_name": "T", + "x_label": "Sequence Length (T)", + "x_values": [2**i for i in range(7, 12)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "B": 4, + "HC": 4, + "C": 4096, + "tmax": 20, + "rms_eps": 1e-6, + "pre_eps": 0.0, + "sinkhorn_eps": 1e-6, + "post_mult": 2.0, + "sub_kernel": sub_kernel, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_mhc, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + + run_benchmarks( + bench_test_fn=bench_memory_mhc, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_mhc_lm.py b/benchmark/scripts/benchmark_mhc_lm.py new file mode 100755 index 0000000..6330a0e --- /dev/null +++ b/benchmark/scripts/benchmark_mhc_lm.py @@ -0,0 +1,455 @@ +import os +import sys + +import torch +import torch.nn as nn +import torch.nn.functional as F +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.mhc import LigerMHC +from liger_kernel.utils import infer_device + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + +device = infer_device() + + +class RMSNorm(nn.Module): + def __init__(self, hidden_size: int, *, eps: float, dtype: torch.dtype, device: str): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype, device=device)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + var = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(var + self.eps) + return x * self.weight + + +def _build_rope_cache(seq_len: int, head_dim: int, *, device: torch.device, dtype: torch.dtype): + inv_freq = 1.0 / (10000 ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim)) + positions = torch.arange(seq_len, device=device, dtype=torch.float32) + freqs = torch.einsum("i,j->ij", positions, inv_freq) + cos = freqs.cos().to(dtype) + sin = freqs.sin().to(dtype) + return cos, sin + + +def _apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + x1 = x[..., ::2] + x2 = x[..., 1::2] + cos = cos[None, None, :, :] + sin = sin[None, None, :, :] + return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1) + + +class MiniLlamaAttention(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, *, dtype: torch.dtype, device: str): + super().__init__() + assert hidden_size % num_heads == 0 + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + assert self.head_dim % 2 == 0, "head_dim must be even for RoPE" + + self.q_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device) + self.k_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device) + self.v_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device) + self.o_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + bsz, seq_len, _ = x.shape + q = self.q_proj(x) + k = self.k_proj(x) + v = self.v_proj(x) + + q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + k = k.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + v = v.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + + cos, sin = _build_rope_cache(seq_len, self.head_dim, device=x.device, dtype=q.dtype) + q = _apply_rope(q, cos, sin) + k = _apply_rope(k, cos, sin) + + attn = F.scaled_dot_product_attention(q, k, v, is_causal=True) + attn = attn.transpose(1, 2).contiguous().view(bsz, seq_len, self.hidden_size) + return self.o_proj(attn) + + +class MiniLlamaMLP(nn.Module): + def __init__(self, hidden_size: int, intermediate_mult: int, *, dtype: torch.dtype, device: str): + super().__init__() + intermediate_size = hidden_size * intermediate_mult + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False, dtype=dtype, device=device) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False, dtype=dtype, device=device) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False, dtype=dtype, device=device) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x)) + + +class AttentionBlock(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, *, dtype: torch.dtype, device: str): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device) + self.attn = MiniLlamaAttention(hidden_size, num_heads, dtype=dtype, device=device) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.attn(self.norm(x)) + + +class MLPBlock(nn.Module): + def __init__(self, hidden_size: int, intermediate_mult: int, *, dtype: torch.dtype, device: str): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device) + self.mlp = MiniLlamaMLP(hidden_size, intermediate_mult, dtype=dtype, device=device) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.mlp(self.norm(x)) + + +class TorchMHC(nn.Module): + def __init__( + self, + layer: nn.Module, + *, + hc: int, + c: int, + tmax: int, + rms_eps: float, + pre_eps: float, + sinkhorn_eps: float, + post_mult: float, + phi_dtype: torch.dtype, + ): + super().__init__() + self.layer = layer + self.hc = int(hc) + self.c = int(c) + self.tmax = int(tmax) + self.rms_eps = float(rms_eps) + self.pre_eps = float(pre_eps) + self.sinkhorn_eps = float(sinkhorn_eps) + self.post_mult = float(post_mult) + + layer_param = next(layer.parameters()) + device = layer_param.device + + m = hc * hc + 2 * hc + k = hc * c + self.phi = nn.Parameter(torch.randn(k, m, dtype=phi_dtype, device=device) * 0.02) + self.b = nn.Parameter(torch.zeros(m, dtype=torch.float32, device=device)) + self.alpha_pre = nn.Parameter(torch.tensor(1.0, dtype=torch.float32, device=device)) + self.alpha_post = nn.Parameter(torch.tensor(1.0, dtype=torch.float32, device=device)) + self.alpha_res = nn.Parameter(torch.tensor(1.0, dtype=torch.float32, device=device)) + + self.layer_dtype = layer_param.dtype + + def _coeffs(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + from test.transformers.test_mhc import mhc_coeffs_ref + + return mhc_coeffs_ref( + x, + self.phi, + self.b, + self.alpha_pre, + self.alpha_post, + self.alpha_res, + tmax=self.tmax, + rms_eps=self.rms_eps, + pre_eps=self.pre_eps, + sinkhorn_eps=self.sinkhorn_eps, + post_mult=self.post_mult, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h_pre, h_post, h_res = self._coeffs(x) + x_in = (x.float() * h_pre.unsqueeze(-1)).sum(dim=-2) + if x_in.dtype != self.layer_dtype: + x_in = x_in.to(self.layer_dtype) + f_out = self.layer(x_in) + x_out = torch.einsum("...oi,...ic->...oc", h_res, x.float()) + h_post.unsqueeze(-1) * f_out.float().unsqueeze( + -2 + ) + return x_out.to(x.dtype) + + +class MHCDecoderLayer(nn.Module): + def __init__( + self, + mhc_cls: type[nn.Module], + *, + hidden_size: int, + hc: int, + num_heads: int, + intermediate_mult: int, + tmax: int, + dtype: torch.dtype, + device: str, + ): + super().__init__() + attn = AttentionBlock(hidden_size, num_heads, dtype=dtype, device=device) + mlp = MLPBlock(hidden_size, intermediate_mult, dtype=dtype, device=device) + self.attn = mhc_cls( + attn, + hc=hc, + c=hidden_size, + tmax=tmax, + rms_eps=1e-6, + pre_eps=1e-4, + sinkhorn_eps=1e-6, + post_mult=2.0, + phi_dtype=dtype, + ) + self.mlp = mhc_cls( + mlp, + hc=hc, + c=hidden_size, + tmax=tmax, + rms_eps=1e-6, + pre_eps=1e-4, + sinkhorn_eps=1e-6, + post_mult=2.0, + phi_dtype=dtype, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.attn(x) + x = self.mlp(x) + return x + + +class BenchMiniMHCLM(nn.Module): + def __init__( + self, + mhc_cls: type[nn.Module], + *, + vocab_size: int, + hidden_size: int, + hc: int, + num_layers: int, + num_heads: int, + intermediate_mult: int, + tmax: int, + dtype: torch.dtype, + device: str, + ): + super().__init__() + self.hc = hc + self.hidden_size = hidden_size + self.embed = nn.Embedding(vocab_size, hc * hidden_size, dtype=dtype, device=device) + self.layers = nn.ModuleList( + [ + MHCDecoderLayer( + mhc_cls, + hidden_size=hidden_size, + hc=hc, + num_heads=num_heads, + intermediate_mult=intermediate_mult, + tmax=tmax, + dtype=dtype, + device=device, + ) + for _ in range(num_layers) + ] + ) + self.final_norm = RMSNorm(hidden_size, eps=1e-6, dtype=dtype, device=device) + self.lm_head = nn.Linear(hidden_size, vocab_size, bias=False, dtype=dtype, device=device) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + x = self.embed(input_ids) + bsz, seq_len, _ = x.shape + x = x.view(bsz, seq_len, self.hc, self.hidden_size) + for layer in self.layers: + x = layer(x) + x = x.mean(dim=-2) + x = self.final_norm(x) + return self.lm_head(x) + + +def _build_model( + provider: str, + *, + hidden_size: int, + hc: int, + num_layers: int, + num_heads: int, + intermediate_mult: int, + vocab_size: int, + tmax: int, + dtype: torch.dtype, +): + mhc_cls = LigerMHC if provider == "liger" else TorchMHC + return BenchMiniMHCLM( + mhc_cls, + vocab_size=vocab_size, + hidden_size=hidden_size, + hc=hc, + num_layers=num_layers, + num_heads=num_heads, + intermediate_mult=intermediate_mult, + tmax=tmax, + dtype=dtype, + device=device, + ) + + +def bench_speed_mhc_lm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + hidden_size = int(input.x) + provider = input.kernel_provider + mode = input.kernel_operation_mode + extra = input.extra_benchmark_config + bsz = extra["B"] + seq_len = extra["T"] + hc = extra["HC"] + num_layers = extra["layers"] + num_heads = extra["heads"] + vocab_size = extra["vocab"] + dtype = extra["dtype"] + tmax = extra["tmax"] + intermediate_mult = extra["intermediate_mult"] + + if hidden_size % num_heads != 0: + raise ValueError("hidden_size must be divisible by num_heads") + + model = _build_model( + provider, + hidden_size=hidden_size, + hc=hc, + num_layers=num_layers, + num_heads=num_heads, + intermediate_mult=intermediate_mult, + vocab_size=vocab_size, + tmax=tmax, + dtype=dtype, + ) + + input_ids = torch.randint(0, vocab_size, (bsz, seq_len), device=device) + + def fwd(): + return model(input_ids) + + def fwd_loss(): + return fwd().float().mean() + + grad_to_none = list(model.parameters()) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, quantiles=QUANTILES, grad_to_none=grad_to_none, rep=100) + elif mode == "backward": + loss = fwd_loss() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: loss.backward(retain_graph=True), + quantiles=QUANTILES, + grad_to_none=grad_to_none, + rep=100, + ) + elif mode == "full": + + def full(): + loss = fwd_loss() + loss.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, grad_to_none=grad_to_none, rep=100) + else: + raise ValueError(f"Unknown mode: {mode}") + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_mhc_lm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + hidden_size = int(input.x) + provider = input.kernel_provider + extra = input.extra_benchmark_config + bsz = extra["B"] + seq_len = extra["T"] + hc = extra["HC"] + num_layers = extra["layers"] + num_heads = extra["heads"] + vocab_size = extra["vocab"] + dtype = extra["dtype"] + tmax = extra["tmax"] + intermediate_mult = extra["intermediate_mult"] + + if hidden_size % num_heads != 0: + raise ValueError("hidden_size must be divisible by num_heads") + + model = _build_model( + provider, + hidden_size=hidden_size, + hc=hc, + num_layers=num_layers, + num_heads=num_heads, + intermediate_mult=intermediate_mult, + vocab_size=vocab_size, + tmax=tmax, + dtype=dtype, + ) + + input_ids = torch.randint(0, vocab_size, (bsz, seq_len), device=device) + + def fwd(): + return model(input_ids) + + def full(): + loss = fwd().float().mean() + loss.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "mhc_llama_like_lm", + "x_name": "hidden_size", + "x_label": "hidden_size", + "x_values": [256, 512, 1024], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "B": 2, + "T": 256, + "HC": 4, + "layers": 2, + "heads": 8, + "vocab": 4096, + "dtype": torch.bfloat16, + "tmax": 8, + "intermediate_mult": 4, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_mhc_lm, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_mhc_lm, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_model_configs.py b/benchmark/scripts/benchmark_model_configs.py new file mode 100755 index 0000000..630b0d5 --- /dev/null +++ b/benchmark/scripts/benchmark_model_configs.py @@ -0,0 +1,258 @@ +""" +Standardized benchmark model configurations. + +Provides canonical model architecture profiles and device-specific benchmark +parameters. All benchmark scripts should derive their tensor shapes from these +shared configs rather than defining ad-hoc per-script constants. + +Usage:: + + from benchmark_model_configs import ( + get_benchmark_model_config, + compute_seq_len_sweep_config, + estimate_kernel_peak_memory, + ) + + args = parse_benchmark_script_args() + model = get_benchmark_model_config(args.model) + + # Measure actual memory via a small probe, then compute sweep config + peak_bytes = estimate_kernel_peak_memory(probe_fn=_probe) + bpt = peak_bytes // probe_num_tokens + config = compute_seq_len_sweep_config(model, kernel_bytes_per_token=bpt) +""" + +import gc +import math + +from dataclasses import dataclass +from typing import Callable +from typing import Dict +from typing import Optional + +import torch + +from liger_kernel.utils import get_total_gpu_memory +from liger_kernel.utils import infer_device + + +@dataclass(frozen=True) +class ModelConfig: + """Canonical model architecture profile. + + Each field corresponds to a standard LLM hyperparameter. Benchmark scripts + pick the fields they need (e.g. hidden_size for RMSNorm, vocab_size for + CrossEntropy) while kernel-specific overrides (e.g. hidden_act for GEGLU) + are applied locally in the benchmark script. + """ + + name: str + hidden_size: int + intermediate_size: int + vocab_size: int + num_attention_heads: int + num_key_value_heads: int + head_dim: int + hidden_act: str + max_position_embeddings: int = 8192 + rms_norm_eps: float = 1e-5 + dtype: torch.dtype = torch.bfloat16 + + +@dataclass(frozen=True) +class SeqLenSweepConfig: + """Config for benchmarks that sweep sequence length (e.g. GEGLU, SwiGLU). + + Attributes: + batch_size: Safe batch size for the sweep. + seq_len: Max sequence length (upper bound for x_values). + """ + + batch_size: int + seq_len: int + + +@dataclass(frozen=True) +class HiddenSizeSweepConfig: + """Config for benchmarks that sweep hidden_size with fixed BT (e.g. DyT). + + Attributes: + bt: Fixed batch * seq dimension. + max_hidden_size: Upper bound for hidden_size sweep. + """ + + bt: int + max_hidden_size: int + + +# ── Model Profiles ────────────────────────────────────────────────────────── + +LLAMA_2_7B = ModelConfig( + name="llama_2_7b", + hidden_size=4096, + intermediate_size=11008, + vocab_size=32000, + num_attention_heads=32, + num_key_value_heads=32, + head_dim=128, + hidden_act="silu", + max_position_embeddings=4096, +) + +LLAMA_3_8B = ModelConfig( + name="llama_3_8b", + hidden_size=4096, + intermediate_size=14336, + vocab_size=128256, + num_attention_heads=32, + num_key_value_heads=8, + head_dim=128, + hidden_act="silu", + max_position_embeddings=8192, +) + +MODEL_REGISTRY: Dict[str, ModelConfig] = { + "llama_2_7b": LLAMA_2_7B, + "llama_3_8b": LLAMA_3_8B, +} + +DEFAULT_MODEL_CONFIG = LLAMA_3_8B + + +def get_benchmark_model_config(model_name: Optional[str] = None) -> ModelConfig: + """Resolve benchmark model config from name. + + Returns the canonical model architecture profile (hidden_size, vocab_size, + dtype, etc.) for benchmark runs. Use this to obtain model attributes + when building benchmark tensors and shapes. + + Args: + model_name: Registry key (e.g. ``llama_2_7b``, ``llama_3_8b``). + If None, returns ``DEFAULT_MODEL_CONFIG``. + """ + return MODEL_REGISTRY[model_name] if model_name else DEFAULT_MODEL_CONFIG + + +def estimate_kernel_peak_memory(probe_fn: Callable[[], torch.Tensor]) -> int: + """Run a forward + backward probe to measure peak memory (bytes). + + Call this with the *pure PyTorch* (e.g. huggingface) implementation -- + that typically has the highest memory footprint and therefore gives a + safe upper-bound estimate. Returns the total peak bytes; divide by + num_tokens if you need bytes-per-token for :func:`compute_seq_len_sweep_config`. + + The probe_fn performs setup and forward pass internally; cleanup is + automatic, so callers do not need to manage tensor/layer lifecycle. + + Example:: + + peak_bytes = estimate_kernel_peak_memory(probe_fn=_probe) + kernel_bpt = peak_bytes // num_tokens # if needed + + Args: + probe_fn: Callable that performs setup, runs a forward pass, and + returns an output tensor suitable for ``.backward()``. + """ + device_str = infer_device() + torch_device_mod = getattr(torch, device_str) + + gc.collect() + torch_device_mod.empty_cache() + torch_device_mod.memory.reset_peak_memory_stats() + + y = probe_fn() + y.backward(torch.randn_like(y)) + + peak_bytes = torch_device_mod.max_memory_allocated() + + del y + gc.collect() + torch_device_mod.empty_cache() + + return max(1, peak_bytes) + + +def compute_seq_len_sweep_config( + model_cfg: ModelConfig, + kernel_bytes_per_token: Optional[int] = None, + memory_utilization: float = 0.4, + max_seq_len: Optional[int] = None, + max_batch_size: int = 32, +) -> SeqLenSweepConfig: + """Compute safe batch_size and seq_len for sequence-length sweep (e.g. GEGLU). + + Peak memory is estimated as + ``batch_size * seq_len * kernel_bytes_per_token`` and is capped at + device memory * memory_utilization. Device memory is obtained + internally via :func:`~liger_kernel.utils.get_total_gpu_memory`. + + Prefer obtaining *kernel_bytes_per_token* via + :func:`estimate_kernel_peak_memory` (divide by num_tokens) rather + than hardcoding an analytical estimate. + + Args: + model_cfg: Model architecture config. + kernel_bytes_per_token: Peak memory **per token** (``batch * seq_len`` + axis). Best obtained from :func:`estimate_kernel_peak_memory` / num_tokens. + Falls back to a conservative heuristic + (``hidden_size * dtype_bytes * 16``) when *None*. + memory_utilization: Fraction of total device memory to target (0 to 1). + Lower values are safer. Default ``0.4`` leaves headroom for + framework overhead and CUDA/NPU context. + max_seq_len: Hard upper bound for sequence length. Defaults to + ``model_cfg.max_position_embeddings`` so the sweep never exceeds + the model's native context window. + max_batch_size: Hard upper bound for batch size. + """ + total_memory_gb = get_total_gpu_memory() + dtype_bytes = 2 if model_cfg.dtype in (torch.bfloat16, torch.float16) else 4 + + if kernel_bytes_per_token is None: + kernel_bytes_per_token = model_cfg.hidden_size * dtype_bytes * 16 + + if max_seq_len is None: + max_seq_len = model_cfg.max_position_embeddings + + usable_bytes = total_memory_gb * (1024**3) * memory_utilization + max_tokens = max(1, int(usable_bytes / kernel_bytes_per_token)) + + seq_len = min(max_seq_len, max_tokens) + seq_len = 2 ** int(math.log2(seq_len)) if seq_len >= 1024 else 1024 + + batch_size = max(1, min(max_tokens // seq_len, max_batch_size)) + + return SeqLenSweepConfig(batch_size=batch_size, seq_len=seq_len) + + +def compute_hidden_size_sweep_config( + model_cfg: ModelConfig, + kernel_peak_bytes: int, + bt: int = 4096, + memory_utilization: float = 0.4, + max_hidden_size_multiplier: int = 4, +) -> HiddenSizeSweepConfig: + """Compute safe max_hidden_size for hidden_size sweep (e.g. DyT). + + For kernels with shape (BT, hidden_size) where BT is fixed and we sweep + hidden_size. Uses probe peak memory to derive max_hidden_size. + Device memory is obtained internally via :func:`~liger_kernel.utils.get_total_gpu_memory`. + + Args: + model_cfg: Model config. + kernel_peak_bytes: Peak memory from probe (BT, model.hidden_size). + bt: Fixed BT dimension; must match the probe. + memory_utilization: Fraction of device memory to use. + max_hidden_size_multiplier: Cap max_hidden_size at model.hidden_size * this. + """ + total_memory_gb = get_total_gpu_memory() + usable_bytes = total_memory_gb * (1024**3) * memory_utilization + kernel_bpt = max(1, kernel_peak_bytes // bt) + max_hidden_size = min( + model_cfg.hidden_size * max_hidden_size_multiplier, + max( + model_cfg.hidden_size, + int(usable_bytes * model_cfg.hidden_size / (bt * kernel_bpt)), + ), + ) + max_hidden_size = max(1024, 2 ** int(math.log2(max_hidden_size))) + return HiddenSizeSweepConfig(bt=bt, max_hidden_size=max_hidden_size) diff --git a/benchmark/scripts/benchmark_multi_token_attention.py b/benchmark/scripts/benchmark_multi_token_attention.py new file mode 100755 index 0000000..b5319af --- /dev/null +++ b/benchmark/scripts/benchmark_multi_token_attention.py @@ -0,0 +1,218 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.multi_token_attention import LigerMultiTokenAttention +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchMultiTokenAttention(torch.nn.Module): + def __init__(self, C_in, C_out, K, groups, bias, dtype, device): + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(C_out, C_in // groups, K, K, dtype=dtype, device=device)) + self.bias = torch.nn.Parameter(torch.empty(C_out, dtype=dtype, device=device)) if bias else None + self.K = K + self.groups = groups + + def forward(self, scores): + B, C_in, L, _ = scores.shape + mask = torch.tril(torch.ones(L, L, dtype=torch.bool, device=scores.device)).view(1, 1, L, L) + inf = torch.tensor(-1e9, device=scores.device, dtype=scores.dtype) + zero = torch.tensor(0.0, device=scores.device, dtype=scores.dtype) + s_inf = scores.masked_fill(~mask, inf) + probs = torch.nn.functional.softmax(s_inf, dim=-1) + out_c = torch.nn.functional.conv2d( + probs, self.weight, self.bias, stride=1, padding=self.K // 2, groups=self.groups + ) + return out_c.masked_fill(~mask, zero) + + +def bench_speed_multi_token_attention(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + L = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + B = extra_benchmark_config["B"] + C_in = extra_benchmark_config["C_in"] + C_out = extra_benchmark_config["C_out"] + K = extra_benchmark_config["K"] + groups = extra_benchmark_config["groups"] + bias = extra_benchmark_config["bias"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (B, C_in, L, L) + + triton_attn = ( + LigerMultiTokenAttention( + in_channels=C_in, + out_channels=C_out, + kernel_size=K, + stride=1, + padding=K // 2, + dilation=1, + groups=groups, + bias=bias, + ) + .to(device) + .to(dtype) + ) + + torch_attn = TorchMultiTokenAttention( + C_in=C_in, C_out=C_out, K=K, groups=groups, bias=bias, dtype=dtype, device=device + ) + + with torch.no_grad(): + torch_attn.weight.copy_(triton_attn.weight) + if bias: + torch_attn.bias.copy_(triton_attn.bias) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def fwd(): + if provider == "liger": + return triton_attn(x) + elif provider == "torch": + return torch_attn(x) + + print(f"Starting Warmup for input size: {x_shape}") + _ = fwd() + if mode in ("backward", "full"): + y = _ + y.backward(dy, retain_graph=True) + print("Done Warmup") + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, grad_to_none=[x], rep=100, quantiles=QUANTILES) + elif mode == "backward": + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + grad_to_none=[x], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, grad_to_none=[x], rep=100, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_multi_token_attention(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + L = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + B = extra_benchmark_config["B"] + C_in = extra_benchmark_config["C_in"] + C_out = extra_benchmark_config["C_out"] + K = extra_benchmark_config["K"] + groups = extra_benchmark_config["groups"] + bias = extra_benchmark_config["bias"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (B, C_in, L, L) + + triton_attn = ( + LigerMultiTokenAttention( + in_channels=C_in, + out_channels=C_out, + kernel_size=K, + stride=1, + padding=K // 2, + dilation=1, + groups=groups, + bias=bias, + ) + .to(device) + .to(dtype) + ) + + torch_attn = TorchMultiTokenAttention( + C_in=C_in, C_out=C_out, K=K, groups=groups, bias=bias, dtype=dtype, device=device + ) + + with torch.no_grad(): + torch_attn.weight.copy_(triton_attn.weight) + if bias: + torch_attn.bias.copy_(triton_attn.bias) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def fwd(): + if provider == "liger": + return triton_attn(x) + elif provider == "torch": + return torch_attn(x) + + def full(): + y = fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "multi_token_attention", + "x_name": "L", + "x_label": "sequence length", + "x_values": [2**i for i in range(5, 10)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "B": 2, + "C_in": 4, + "C_out": 4, + "K": 3, + "groups": 1, + "bias": True, + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_multi_token_attention, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_multi_token_attention, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_orpo_loss.py b/benchmark/scripts/benchmark_orpo_loss.py new file mode 100755 index 0000000..30b308c --- /dev/null +++ b/benchmark/scripts/benchmark_orpo_loss.py @@ -0,0 +1,169 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +############################################################################# +# Test the memory consumption of the linear fused cross entropy loss +############################################################################# + + +def bench_memory_fused_linear_orpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_orpo_loss import LigerLMHeadORPO + from test.chunked_loss.test_orpo_loss import TorchLMHeadORPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + + # Instantiate once and retrieve the first output only + torch_lm_head_orpo = TorchLMHeadORPO(H=H, V=V, dtype=dtype).to(device) + liger_lm_head_orpo = LigerLMHeadORPO(H=H, V=V, dtype=dtype).to(device) + torch_fwd = lambda x, target, nll_target: torch_lm_head_orpo(x, target, nll_target)[0] + liger_fwd = lambda x, target, nll_target: liger_lm_head_orpo(x, target, nll_target)[0] + + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + nll_target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + def fwd(): + if provider == "liger": + return liger_fwd(_input, target, nll_target) + elif provider == "huggingface": + return torch_fwd(_input, target, nll_target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +# ############################################################################# +# # Test the speed of the fused linear cross entropy loss +# ############################################################################# + + +def bench_speed_fused_linear_orpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_orpo_loss import LigerLMHeadORPO + from test.chunked_loss.test_orpo_loss import TorchLMHeadORPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + # Instantiate once and retrieve the first output only + torch_lm_head_orpo = TorchLMHeadORPO(H=H, V=V, dtype=dtype).to(device) + liger_lm_head_orpo = LigerLMHeadORPO(H=H, V=V, dtype=dtype).to(device) + torch_fwd = lambda x, target, nll_target: torch_lm_head_orpo(x, target, nll_target)[0] + liger_fwd = lambda x, target, nll_target: liger_lm_head_orpo(x, target, nll_target)[0] + + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + nll_target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + def fwd(): + if provider == "liger": + return liger_fwd(_input, target, nll_target) + elif provider == "huggingface": + return torch_fwd(_input, target, nll_target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_linear_orpo_loss", + "x_name": "B", + "x_label": "B", + "x_values": [2**i for i in range(1, 5)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "T": 1024, + "H": 4096, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_orpo_loss, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_orpo_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_poly_norm.py b/benchmark/scripts/benchmark_poly_norm.py new file mode 100755 index 0000000..ddff431 --- /dev/null +++ b/benchmark/scripts/benchmark_poly_norm.py @@ -0,0 +1,197 @@ +import torch +import torch.nn as nn +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.poly_norm import LigerPolyNorm +from liger_kernel.utils import infer_device + +device = infer_device() + + +class NaivePolyNorm(nn.Module): + """ + Naive PyTorch implementation of PolyNorm. + + Reference: + https://github.com/BryceZhuo/PolyCom/ + + PolyNorm formula: + y = w₀·norm(x³) + w₁·norm(x²) + w₂·norm(x) + b + where norm(u) = u / sqrt(mean(u²) + ε) + """ + + def __init__(self, eps=1e-6): + super().__init__() + # Align with PolyCom reference: (1/3, 1/3, 1/3) and bias=1.0 + self.weight = nn.Parameter(torch.full((3,), 1.0 / 3.0)) + self.bias = nn.Parameter(torch.tensor(1.0)) + self.variance_epsilon = eps + + def _norm(self, x): + """RMSNorm operation""" + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) + + def forward(self, hidden_states): + """ + Forward pass of PolyNorm + + Args: + hidden_states: input tensor of shape (..., H) + + Returns: + output tensor of same shape as input + """ + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + + # Compute powers + x_pow3 = hidden_states**3 + x_pow2 = hidden_states**2 + x_pow1 = hidden_states**1 + + # Normalize each power + norm_x3 = self._norm(x_pow3) + norm_x2 = self._norm(x_pow2) + norm_x1 = self._norm(x_pow1) + + # Weighted sum with bias + output = self.weight[0] * norm_x3 + self.weight[1] * norm_x2 + self.weight[2] * norm_x1 + self.bias + + return output.to(input_dtype) + + +def bench_speed_poly_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + + triton_poly = LigerPolyNorm(eps=eps).to(device) + naive_poly = NaivePolyNorm(eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + # utility functions + + def y_fwd(): + if provider == "liger": + return triton_poly(x) + + if provider == "huggingface": + return naive_poly(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + y_fwd, + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_poly_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + + triton_poly = LigerPolyNorm(eps=eps).to(device) + naive_poly = NaivePolyNorm(eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + # utility functions + def y_fwd(): + if provider == "liger": + return triton_poly(x) + if provider == "huggingface": + return naive_poly(x) + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "poly_norm", + "x_name": "H", + "x_label": "hidden size", + "x_values": [2**i for i in range(10, 16)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [{"M": 2048, "dtype": torch.bfloat16, "eps": 1e-6}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_poly_norm, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_poly_norm, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_qwen2vl_mrope.py b/benchmark/scripts/benchmark_qwen2vl_mrope.py new file mode 100755 index 0000000..ec1c53b --- /dev/null +++ b/benchmark/scripts/benchmark_qwen2vl_mrope.py @@ -0,0 +1,241 @@ +import torch +import triton + +from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLTextConfig +from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLRotaryEmbedding +from transformers.models.qwen2_vl.modeling_qwen2_vl import apply_multimodal_rotary_pos_emb +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.qwen2vl_mrope import liger_multimodal_rotary_pos_emb +from liger_kernel.utils import infer_device + +device = infer_device() + + +def bench_speed_qwen2vl_mrope( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + num_q_heads = extra_benchmark_config["num_q_heads"] + num_kv_heads = extra_benchmark_config["num_kv_heads"] + dtype = extra_benchmark_config["dtype"] + + # x can be either hidden_size or seq_len + hidden_size = extra_benchmark_config["hidden_size"] if "hidden_size" in extra_benchmark_config else input.x + seq_len = extra_benchmark_config["seq_len"] if "seq_len" in extra_benchmark_config else input.x + + head_dim = hidden_size // num_q_heads + mrope_section_hw = head_dim * 3 // 16 + mrope_section = [ + head_dim // 2 - 2 * mrope_section_hw, + mrope_section_hw, + mrope_section_hw, + ] + config = Qwen2VLTextConfig( + hidden_size=hidden_size, + num_attention_heads=num_q_heads, + num_key_value_heads=num_kv_heads, + rope_theta=1000000.0, + mrope_section=mrope_section, + ) + rotary_emb = Qwen2VLRotaryEmbedding(config, device=device) + q = torch.randn( + (1, seq_len, num_q_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + k = torch.randn( + (1, seq_len, num_kv_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + dq, dk = ( + torch.randn_like(q, device=device, dtype=dtype), + torch.randn_like(k, device=device, dtype=dtype), + ) + pos_ids = torch.arange(seq_len * 3, device=device, dtype=torch.long).view(3, 1, -1) + cos, sin = rotary_emb(k, pos_ids) + + def fwd(): + if provider == "liger": + return liger_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section) + elif provider == "huggingface": + return apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section) + else: + raise ValueError(f"Invalid provider: {provider} for M-RoPE embedding") + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + elif mode == "backward": + q_out, k_out = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True, retain_graph=True), + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + q_out, k_out = fwd() + torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_qwen2vl_mrope( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + num_q_heads = extra_benchmark_config["num_q_heads"] + num_kv_heads = extra_benchmark_config["num_kv_heads"] + dtype = extra_benchmark_config["dtype"] + + # x can be either hidden_size or seq_len + hidden_size = extra_benchmark_config["hidden_size"] if "hidden_size" in extra_benchmark_config else input.x + seq_len = extra_benchmark_config["seq_len"] if "seq_len" in extra_benchmark_config else input.x + + head_dim = hidden_size // num_q_heads + + mrope_section_hw = head_dim * 3 // 16 + mrope_section = [ + head_dim // 2 - 2 * mrope_section_hw, + mrope_section_hw, + mrope_section_hw, + ] + config = Qwen2VLTextConfig( + hidden_size=hidden_size, + num_attention_heads=num_q_heads, + num_key_value_heads=num_kv_heads, + rope_theta=1000000.0, + mrope_section=mrope_section, + ) + rotary_emb = Qwen2VLRotaryEmbedding(config, device=device) + q = torch.randn( + (1, seq_len, num_q_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + k = torch.randn( + (1, seq_len, num_kv_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + dq, dk = ( + torch.randn_like(q, device=device, dtype=dtype), + torch.randn_like(k, device=device, dtype=dtype), + ) + pos_ids = torch.arange(seq_len * 3, device=device, dtype=torch.long).view(3, 1, -1) + cos, sin = rotary_emb(k, pos_ids) + + def full(): + if provider == "liger": + q_out, k_out = liger_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section) + else: + q_out, k_out = apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section) + torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory( + full, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs_varying_hidden_size = { + "kernel_name": "qwen2vl_mrope", + "x_name": "H", + "x_label": "hidden size", + "x_values": [32 * (2**i) for i in range(4, 10, 2)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "dtype": torch.bfloat16, + "seq_len": 2048, + "num_q_heads": 32, + "num_kv_heads": 8, + } + ], + "overwrite": args.overwrite, + } + run_benchmarks( + bench_test_fn=bench_speed_qwen2vl_mrope, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs_varying_hidden_size, + ) + run_benchmarks( + bench_test_fn=bench_memory_qwen2vl_mrope, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs_varying_hidden_size, + ) + + common_configs_varying_seq_len = { + "kernel_name": "qwen2vl_mrope", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, 15)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "dtype": torch.bfloat16, + "hidden_size": 8192, + "num_q_heads": 32, + "num_kv_heads": 8, + } + ], + "overwrite": args.overwrite, + } + run_benchmarks( + bench_test_fn=bench_speed_qwen2vl_mrope, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs_varying_seq_len, + ) + run_benchmarks( + bench_test_fn=bench_memory_qwen2vl_mrope, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs_varying_seq_len, + ) diff --git a/benchmark/scripts/benchmark_rms_norm.py b/benchmark/scripts/benchmark_rms_norm.py new file mode 100755 index 0000000..6bcd56a --- /dev/null +++ b/benchmark/scripts/benchmark_rms_norm.py @@ -0,0 +1,162 @@ +import torch +import torch.nn as nn +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.rms_norm import LigerRMSNorm +from liger_kernel.utils import infer_device + +device = infer_device() + + +class LlamaRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +def bench_speed_rms_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + + triton_rms = LigerRMSNorm(hidden_size=N, eps=eps).to(device) + llama_rms = LlamaRMSNorm(hidden_size=N, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + # utility functions + + def y_fwd(): + if provider == "liger": + return triton_rms(x) + + if provider == "huggingface": + return llama_rms(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + y_fwd, + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_rms_norm(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + eps = extra_benchmark_config["eps"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + + triton_rms = LigerRMSNorm(hidden_size=N, eps=eps).to(device) + llama_rms = LlamaRMSNorm(hidden_size=N, eps=eps).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + # utility functions + def y_fwd(): + if provider == "liger": + return triton_rms(x) + if provider == "huggingface": + return llama_rms(x) + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "rms_norm", + "x_name": "H", + "x_label": "hidden size", + "x_values": [2**i for i in range(10, 16)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [{"M": 2048, "dtype": torch.bfloat16, "eps": 1e-6}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_rms_norm, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_rms_norm, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_rope.py b/benchmark/scripts/benchmark_rope.py new file mode 100755 index 0000000..ac79288 --- /dev/null +++ b/benchmark/scripts/benchmark_rope.py @@ -0,0 +1,223 @@ +import torch +import triton + +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding +from transformers.models.llama.modeling_llama import apply_rotary_pos_emb +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.rope import liger_rotary_pos_emb +from liger_kernel.utils import infer_device +from liger_kernel.utils import transformers_version_dispatch + +device = infer_device() + + +def bench_speed_rope(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + num_q_heads = extra_benchmark_config["num_q_heads"] + num_kv_heads = extra_benchmark_config["num_kv_heads"] + dtype = extra_benchmark_config["dtype"] + + # x can be either hidden_size or seq_len + hidden_size = extra_benchmark_config["hidden_size"] if "hidden_size" in extra_benchmark_config else input.x + seq_len = extra_benchmark_config["seq_len"] if "seq_len" in extra_benchmark_config else input.x + + head_dim = hidden_size // num_q_heads + rotary_emb = transformers_version_dispatch( + "4.48.0", + LlamaRotaryEmbedding, + LlamaRotaryEmbedding, + before_kwargs={"dim": head_dim, "device": device}, + after_kwargs={"config": LlamaConfig(num_kv_heads=num_kv_heads, head_dim=head_dim), "device": device}, + ) + q = torch.randn( + (1, seq_len, num_q_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + k = torch.randn( + (1, seq_len, num_kv_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + dq, dk = ( + torch.randn_like(q, device=device, dtype=dtype), + torch.randn_like(k, device=device), + ) + pos_ids = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0) + cos, sin = rotary_emb(k, pos_ids) + + def fwd(): + if provider == "liger": + return liger_rotary_pos_emb(q, k, cos, sin, pos_ids) + elif provider == "huggingface": + return apply_rotary_pos_emb(q, k, cos, sin, pos_ids) + else: + raise ValueError(f"Invalid provider: {provider} for RoPE embedding") + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + elif mode == "backward": + q_out, k_out = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True, retain_graph=True), + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + q_out, k_out = fwd() + torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[q, k], + rep=400, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_rope(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + num_q_heads = extra_benchmark_config["num_q_heads"] + num_kv_heads = extra_benchmark_config["num_kv_heads"] + dtype = extra_benchmark_config["dtype"] + + # x can be either hidden_size or seq_len + hidden_size = extra_benchmark_config["hidden_size"] if "hidden_size" in extra_benchmark_config else input.x + seq_len = extra_benchmark_config["seq_len"] if "seq_len" in extra_benchmark_config else input.x + + head_dim = hidden_size // num_q_heads + rotary_emb = transformers_version_dispatch( + "4.48.0", + LlamaRotaryEmbedding, + LlamaRotaryEmbedding, + before_kwargs={"dim": head_dim, "device": device}, + after_kwargs={"config": LlamaConfig(num_kv_heads=num_kv_heads, head_dim=head_dim), "device": device}, + ) + q = torch.randn( + (1, seq_len, num_q_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + k = torch.randn( + (1, seq_len, num_kv_heads, head_dim), + device=device, + requires_grad=True, + dtype=dtype, + ).transpose(1, 2) + dq, dk = ( + torch.randn_like(q, device=device, dtype=dtype), + torch.randn_like(k, device=device), + ) + pos_ids = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0) + cos, sin = rotary_emb(k, pos_ids) + + def full(): + if provider == "liger": + q_out, k_out = liger_rotary_pos_emb(q, k, cos, sin, pos_ids) + else: + q_out, k_out = apply_rotary_pos_emb(q, k, cos, sin, pos_ids) + torch.autograd.grad((q_out, k_out), (q, k), (dq, dk), allow_unused=True, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory( + full, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs_varying_hidden_size = { + "kernel_name": "rope", + "x_name": "H", + "x_label": "hidden size", + "x_values": [32 * (2**i) for i in range(4, 10, 2)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "dtype": torch.bfloat16, + "seq_len": 2048, + "num_q_heads": 32, + "num_kv_heads": 8, + } + ], + "overwrite": args.overwrite, + } + run_benchmarks( + bench_test_fn=bench_speed_rope, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs_varying_hidden_size, + ) + run_benchmarks( + bench_test_fn=bench_memory_rope, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs_varying_hidden_size, + ) + + common_configs_varying_seq_len = { + "kernel_name": "rope", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, 15)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "dtype": torch.bfloat16, + "hidden_size": 8192, + "num_q_heads": 32, + "num_kv_heads": 8, + } + ], + "overwrite": args.overwrite, + } + run_benchmarks( + bench_test_fn=bench_speed_rope, + kernel_operation_modes=["forward", "backward", "full"], + metric_name="speed", + metric_unit="ms", + **common_configs_varying_seq_len, + ) + run_benchmarks( + bench_test_fn=bench_memory_rope, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs_varying_seq_len, + ) diff --git a/benchmark/scripts/benchmark_simpo_loss.py b/benchmark/scripts/benchmark_simpo_loss.py new file mode 100755 index 0000000..148b8e3 --- /dev/null +++ b/benchmark/scripts/benchmark_simpo_loss.py @@ -0,0 +1,167 @@ +import os +import sys + +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.utils import infer_device + +device = infer_device() + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + + +############################################################################# +# Test the memory consumption of the linear fused cross entropy loss +############################################################################# + + +def bench_memory_fused_linear_simpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_simpo_loss import LigerLMHeadSimPO + from test.chunked_loss.test_simpo_loss import TorchLMHeadCPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + + # Instantiate once and retrieve the first output only + torch_lm_head_simpo = TorchLMHeadCPO(H=H, V=V, dtype=dtype).to(device) + liger_lm_head_simpo = LigerLMHeadSimPO(H=H, V=V, dtype=dtype).to(device) + torch_fwd = lambda x, target: torch_lm_head_simpo(x, target)[0] + liger_fwd = lambda x, target: liger_lm_head_simpo(x, target)[0] + + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + def fwd(): + if provider == "liger": + return liger_fwd(_input, target) + elif provider == "huggingface": + return torch_fwd(_input, target) + + def full(): + y = fwd() + y.backward() + + mem_50, mem_20, mem_80 = _test_memory(full, _iter=10, quantiles=QUANTILES) + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +# ############################################################################# +# # Test the speed of the fused linear cross entropy loss +# ############################################################################# + + +def bench_speed_fused_linear_simpo_loss( + input: SingleBenchmarkRunInput, +) -> SingleBenchmarkRunOutput: + from test.chunked_loss.test_simpo_loss import LigerLMHeadSimPO + from test.chunked_loss.test_simpo_loss import TorchLMHeadCPO + + B = input.x + T = input.extra_benchmark_config["T"] + H = input.extra_benchmark_config["H"] + V = input.extra_benchmark_config["V"] + dtype = input.extra_benchmark_config["dtype"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + # Instantiate once and retrieve the first output only + torch_lm_head_simpo = TorchLMHeadCPO(H=H, V=V, dtype=dtype).to(device) + liger_lm_head_simpo = LigerLMHeadSimPO(H=H, V=V, dtype=dtype).to(device) + torch_fwd = lambda x, target: torch_lm_head_simpo(x, target)[0] + liger_fwd = lambda x, target: liger_lm_head_simpo(x, target)[0] + + _input = torch.randn(B, T, H, requires_grad=True, dtype=dtype, device=device) + target = torch.randint(V, (B, T), dtype=torch.long, device=device) + + def fwd(): + if provider == "liger": + return liger_fwd(_input, target) + elif provider == "huggingface": + return torch_fwd(_input, target) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + rep=100, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + grad_to_none=[_input], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + rep=100, + quantiles=QUANTILES, + ) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "fused_linear_simpo_loss", + "x_name": "B", + "x_label": "B", + "x_values": [2**i for i in range(1, 5)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "T": 1024, + "H": 4096, + "V": 128256, + "mode": "forward", + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_fused_linear_simpo_loss, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_fused_linear_simpo_loss, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_softmax.py b/benchmark/scripts/benchmark_softmax.py new file mode 100755 index 0000000..10e994c --- /dev/null +++ b/benchmark/scripts/benchmark_softmax.py @@ -0,0 +1,140 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.softmax import LigerSoftmax +from liger_kernel.utils import infer_device + +device = infer_device() + + +def bench_speed_softmax(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + N = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + extra_benchmark_config = input.extra_benchmark_config + M = extra_benchmark_config["M"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (M, N) + liger_softmax = LigerSoftmax().to(device).to(dtype) + torch_softmax = torch.nn.Softmax(dim=-1).to(device).to(dtype) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def y_fwd(): + if provider == "liger": + return liger_softmax(x) + if provider == "torch": + return torch_softmax(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(y_fwd, quantiles=QUANTILES, grad_to_none=[x], rep=500) + elif mode == "backward": + y = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[x], + rep=500, + ) + elif mode == "full": + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, grad_to_none=[x], rep=500) + + if any(val is None for val in (ms_20, ms_50, ms_80)): + raise RuntimeError(f"Benchmark speed result is None: ms_20={ms_20}, ms_50={ms_50}, ms_80={ms_80}") + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_softmax(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + shape = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + extra_benchmark_config = input.extra_benchmark_config + dtype = extra_benchmark_config.get("dtype", torch.float32) + + torch_softmax = torch.nn.Softmax(dim=-1) + liger_softmax = LigerSoftmax().to(device).to(dtype) + + x = torch.randn(shape, device=device, dtype=dtype, requires_grad=True) + + def fwd(): + if provider == "liger": + return liger_softmax(x) + elif provider == "torch": + return torch_softmax(x) + else: + raise ValueError(f"Invalid provider: {provider} for softmax") + + def full(): + y = fwd() + y.backward(torch.ones_like(y), retain_graph=True) + + if mode == "forward": + mem_50, mem_20, mem_80 = _test_memory(fwd, quantiles=QUANTILES) + elif mode == "backward": + do = torch.ones_like(x) + y = fwd() + mem_50, mem_20, mem_80 = _test_memory(lambda: y.backward(do, retain_graph=True), quantiles=QUANTILES) + else: + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + if any(val is None for val in (mem_20, mem_50, mem_80)): + raise RuntimeError(f"Benchmark memory result is None: mem_20={mem_20}, mem_50={mem_50}, mem_80={mem_80}") + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = dict( + kernel_name="softmax", + x_name="N", + x_label="hidden size", + x_values=[128, 256, 512, 1024, 2048, 4096], + kernel_providers=["liger", "torch"], + extra_benchmark_configs=[ + {"M": 2048, "dtype": torch.float32}, + {"M": 2048, "dtype": torch.bfloat16}, + ], + ) + + run_benchmarks( + bench_test_fn=bench_speed_softmax, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + overwrite=args.overwrite, + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_softmax, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + overwrite=args.overwrite, + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_sparse_multi_token_attention.py b/benchmark/scripts/benchmark_sparse_multi_token_attention.py new file mode 100755 index 0000000..98f47d7 --- /dev/null +++ b/benchmark/scripts/benchmark_sparse_multi_token_attention.py @@ -0,0 +1,254 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.multi_token_attention import LigerMultiTokenAttention +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchSparseMultiTokenAttention(torch.nn.Module): + def __init__(self, C_in, C_out, K, groups, bias, dtype, device): + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(C_out, C_in // groups, K, K, dtype=dtype, device=device)) + self.bias = torch.nn.Parameter(torch.empty(C_out, dtype=dtype, device=device)) if bias else None + self.K = K + self.groups = groups + self.dtype = dtype + self.compute_dtype = torch.float32 + + def forward(self, scores): + B, C_in, L, _ = scores.shape + mask = torch.tril(torch.ones(L, L, dtype=torch.bool, device=scores.device)).view(1, 1, L, L) + inf = torch.tensor(-1e9, device=scores.device, dtype=self.compute_dtype) + zero = torch.tensor(0.0, device=scores.device, dtype=self.compute_dtype) + + s_compute = scores.to(self.compute_dtype) + s_inf = s_compute.masked_fill(~mask, inf) + + dim = -1 + z = s_inf + + z_sorted, _ = torch.sort(z, dim=dim, descending=True) + + cum_sum = torch.cumsum(z_sorted, dim=dim) + + k_indices = torch.arange(1, L + 1, device=z.device, dtype=z.dtype).view(1, 1, 1, L) + + is_positive = z_sorted > -1e8 + condition = (1 + k_indices * z_sorted > cum_sum) & is_positive + k_sparsemax = torch.sum(condition, dim=dim, keepdim=True) + + k_sparsemax_safe = torch.max(k_sparsemax, torch.ones_like(k_sparsemax)) + + cum_sum_k = torch.gather(cum_sum, dim=dim, index=k_sparsemax_safe.long() - 1) + + tau = (cum_sum_k - 1) / k_sparsemax_safe.to(z.dtype) + tau = torch.where(k_sparsemax == 0, torch.full_like(tau, float("inf")), tau) + + probs = torch.clamp(z - tau, min=0) + + weight_compute = self.weight.to(self.compute_dtype) + bias_compute = self.bias.to(self.compute_dtype) if self.bias is not None else None + + out_c = torch.nn.functional.conv2d( + probs, weight_compute, bias_compute, stride=1, padding=self.K // 2, groups=self.groups + ) + return out_c.masked_fill(~mask, zero).to(scores.dtype) + + +def bench_speed_sparse_multi_token_attention(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + L = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + B = extra_benchmark_config["B"] + C_in = extra_benchmark_config["C_in"] + C_out = extra_benchmark_config["C_out"] + K = extra_benchmark_config["K"] + groups = extra_benchmark_config["groups"] + bias = extra_benchmark_config["bias"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (B, C_in, L, L) + + liger_attn = ( + LigerMultiTokenAttention( + in_channels=C_in, + out_channels=C_out, + kernel_size=K, + stride=1, + padding=K // 2, + dilation=1, + groups=groups, + bias=bias, + sparse=True, + ) + .to(device) + .to(dtype) + ) + + torch_attn = TorchSparseMultiTokenAttention( + C_in=C_in, C_out=C_out, K=K, groups=groups, bias=bias, dtype=dtype, device=device + ) + + with torch.no_grad(): + torch.nn.init.kaiming_uniform_(liger_attn.weight, a=5**0.5) + if bias: + torch.nn.init.zeros_(liger_attn.bias) + torch_attn.weight.copy_(liger_attn.weight) + if bias: + torch_attn.bias.copy_(liger_attn.bias) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def fwd(): + if provider == "liger": + return liger_attn(x) + elif provider == "torch": + return torch_attn(x) + + print(f"Starting Warmup for input size: {x_shape}") + _ = fwd() + if mode in ("backward", "full"): + y = _ + y.backward(dy, retain_graph=True) + print("Done Warmup") + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, grad_to_none=[x], rep=100, quantiles=QUANTILES) + elif mode == "backward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + grad_to_none=[x], + rep=100, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, grad_to_none=[x], rep=100, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_sparse_multi_token_attention(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + L = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + B = extra_benchmark_config["B"] + C_in = extra_benchmark_config["C_in"] + C_out = extra_benchmark_config["C_out"] + K = extra_benchmark_config["K"] + groups = extra_benchmark_config["groups"] + bias = extra_benchmark_config["bias"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (B, C_in, L, L) + + liger_attn = ( + LigerMultiTokenAttention( + in_channels=C_in, + out_channels=C_out, + kernel_size=K, + stride=1, + padding=K // 2, + dilation=1, + groups=groups, + bias=bias, + sparse=True, + ) + .to(device) + .to(dtype) + ) + + torch_attn = TorchSparseMultiTokenAttention( + C_in=C_in, C_out=C_out, K=K, groups=groups, bias=bias, dtype=dtype, device=device + ) + + with torch.no_grad(): + torch.nn.init.kaiming_uniform_(liger_attn.weight, a=5**0.5) + if bias: + torch.nn.init.zeros_(liger_attn.bias) + torch_attn.weight.copy_(liger_attn.weight) + if bias: + torch_attn.bias.copy_(liger_attn.bias) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + def fwd(): + if provider == "liger": + return liger_attn(x) + elif provider == "torch": + return torch_attn(x) + + def full(): + y = fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "sparse_multi_token_attention", + "x_name": "L", + "x_label": "sequence length", + "x_values": [2**i for i in range(5, 10)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [ + { + "B": 2, + "C_in": 4, + "C_out": 4, + "K": 3, + "groups": 1, + "bias": True, + "dtype": torch.float32, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_sparse_multi_token_attention, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_sparse_multi_token_attention, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_sparsemax.py b/benchmark/scripts/benchmark_sparsemax.py new file mode 100755 index 0000000..919f4c6 --- /dev/null +++ b/benchmark/scripts/benchmark_sparsemax.py @@ -0,0 +1,172 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.sparsemax import LigerSparsemax +from liger_kernel.utils import infer_device + +device = infer_device() + + +def torch_sparsemax(input_tensor: torch.Tensor, dim: int = -1) -> torch.Tensor: + input_dims = input_tensor.dim() + if dim < 0: + dim = input_dims + dim + input_sorted, _ = torch.sort(input_tensor, dim=dim, descending=True) + cumsum_input = torch.cumsum(input_sorted, dim=dim) + input_size = input_tensor.size(dim) + range_tensor = torch.arange(1, input_size + 1, device=input_tensor.device, dtype=input_tensor.dtype) + shape = [1] * input_dims + shape[dim] = input_size + range_tensor = range_tensor.view(shape) + k_bound = 1 + range_tensor * input_sorted + support = k_bound > cumsum_input + k = support.sum(dim=dim, keepdim=True).clamp(min=1) + support_sum = (input_sorted * support).sum(dim=dim, keepdim=True) + tau = (support_sum - 1) / k + return torch.clamp(input_tensor - tau, min=0) + + +class TorchSparsemax(torch.nn.Module): + def __init__(self, dim: int = -1): + super().__init__() + self.dim = dim + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return torch_sparsemax(x, dim=self.dim) + + +def bench_speed_sparsemax(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + V = input.x + provider = input.kernel_provider + mode = input.kernel_operation_mode + + extra_benchmark_config = input.extra_benchmark_config + B = extra_benchmark_config["B"] + T = extra_benchmark_config["T"] + dim = extra_benchmark_config["dim"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (B * T, V) + + torch_sparsemax_module = TorchSparsemax(dim=dim).to(device) + liger_sparsemax_module = LigerSparsemax(dim=dim).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + # utility functions + def y_fwd(): + if provider == "liger": + return liger_sparsemax_module(x) + elif provider == "torch": + return torch_sparsemax_module(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + y_fwd, + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = y_fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(dy, retain_graph=True), + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[x], + rep=500, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_sparsemax(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + V = input.x + provider = input.kernel_provider + + extra_benchmark_config = input.extra_benchmark_config + B = extra_benchmark_config["B"] + T = extra_benchmark_config["T"] + dim = extra_benchmark_config["dim"] + dtype = extra_benchmark_config["dtype"] + + x_shape = (B * T, V) + + torch_sparsemax_module = TorchSparsemax(dim=dim).to(device) + liger_sparsemax_module = LigerSparsemax(dim=dim).to(device) + + x = torch.randn(x_shape, dtype=dtype, device=device) + dy = torch.randn_like(x) + x.requires_grad_(True) + + # utility functions + def y_fwd(): + if provider == "liger": + return liger_sparsemax_module(x) + elif provider == "torch": + return torch_sparsemax_module(x) + + def full(): + y = y_fwd() + y.backward(dy, retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + common_configs = { + "kernel_name": "sparsemax", + "x_name": "V", + "x_label": "feature size", + "x_values": [2**i for i in range(10, 16)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [{"B": 4, "T": 512, "dim": -1, "dtype": torch.float32}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_sparsemax, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_sparsemax, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_swiglu.py b/benchmark/scripts/benchmark_swiglu.py new file mode 100755 index 0000000..8d46572 --- /dev/null +++ b/benchmark/scripts/benchmark_swiglu.py @@ -0,0 +1,115 @@ +import math + +import torch + +from benchmark_model_configs import compute_seq_len_sweep_config +from benchmark_model_configs import estimate_kernel_peak_memory +from benchmark_model_configs import get_benchmark_model_config +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import LlamaMLP +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import parse_benchmark_script_args +from utils import run_benchmarks +from utils import run_memory_benchmark +from utils import run_speed_benchmark + +from liger_kernel.transformers.swiglu import LigerSwiGLUMLP +from liger_kernel.utils import infer_device + +device = infer_device() + + +def _setup_swiglu(input: SingleBenchmarkRunInput): + """Create input tensor and SwiGLU layer from benchmark config.""" + cfg = input.extra_benchmark_config + llama_config = LlamaConfig( + hidden_size=cfg["hidden_size"], + intermediate_size=cfg["intermediate_size"], + hidden_act=cfg["hidden_act"], + ) + x = torch.randn( + cfg["bsz"], + input.x, + cfg["hidden_size"], + device=device, + dtype=cfg["dtype"], + requires_grad=True, + ) + if input.kernel_provider == "liger": + layer = LigerSwiGLUMLP(config=llama_config).to(device).to(cfg["dtype"]) + elif input.kernel_provider == "huggingface": + layer = LlamaMLP(config=llama_config).to(device).to(cfg["dtype"]) + else: + raise ValueError(f"Invalid provider: {input.kernel_provider} for SwiGLU") + return x, layer + + +def bench_speed_swiglu(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_swiglu(input) + return run_speed_benchmark(lambda: layer(x), input.kernel_operation_mode, [x]) + + +def bench_memory_swiglu(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + x, layer = _setup_swiglu(input) + return run_memory_benchmark(lambda: layer(x), input.kernel_operation_mode) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + model = get_benchmark_model_config(args.model) + probe_seq_len = 1024 + + def _probe(): + probe_input = SingleBenchmarkRunInput( + x=probe_seq_len, + kernel_provider="huggingface", + extra_benchmark_config={ + "bsz": 1, + "hidden_size": model.hidden_size, + "intermediate_size": model.intermediate_size, + "hidden_act": "silu", + "dtype": model.dtype, + }, + ) + x, layer = _setup_swiglu(probe_input) + return layer(x) + + peak_bytes = estimate_kernel_peak_memory(probe_fn=_probe) + kernel_bpt = peak_bytes // probe_seq_len + + config = compute_seq_len_sweep_config(model, kernel_bytes_per_token=kernel_bpt) + + common_configs = { + "kernel_name": "swiglu", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, int(math.log2(config.seq_len)) + 1)], + "kernel_providers": ["liger", "huggingface"], + "extra_benchmark_configs": [ + { + "bsz": config.batch_size, + "hidden_size": model.hidden_size, + "intermediate_size": model.intermediate_size, + "hidden_act": "silu", + "dtype": model.dtype, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_swiglu, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs, + ) + run_benchmarks( + bench_test_fn=bench_memory_swiglu, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="memory", + metric_unit="MB", + **common_configs, + ) diff --git a/benchmark/scripts/benchmark_tiled_mlp.py b/benchmark/scripts/benchmark_tiled_mlp.py new file mode 100755 index 0000000..1eaf21d --- /dev/null +++ b/benchmark/scripts/benchmark_tiled_mlp.py @@ -0,0 +1,397 @@ +import math + +import torch +import torch.nn as nn +import triton + +from transformers.models.llama.configuration_llama import LlamaConfig +from transformers.models.llama.modeling_llama import LlamaMLP +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.geglu import LigerGEGLUMLP +from liger_kernel.transformers.swiglu import LigerSwiGLUMLP +from liger_kernel.transformers.tiled_mlp import LigerTiledGEGLUMLP +from liger_kernel.transformers.tiled_mlp import LigerTiledSwiGLUMLP +from liger_kernel.utils import infer_device + +device = infer_device() + + +# DeepSpeed TiledMLP implementation +# Based on: https://github.com/deepspeedai/DeepSpeed/blob/v0.18.2/deepspeed/runtime/sequence_parallel/ulysses_sp.py#L838 +class DeepSpeedTiledMLP(torch.autograd.Function): + """ + DeepSpeed's TiledMLP implementation for fair comparison. + This is the actual DeepSpeed algorithm that performs tiled MLP computation + to massively reduce memory usage with very long sequence lengths. + + This module re-computes forward in the backward, so forward occurs twice per iteration. + """ + + @staticmethod + def forward(ctx, fn, self, x, shards, compute_params) -> torch.Tensor: + ctx.fn = fn + ctx.self = self + ctx.shards = shards + ctx.compute_params = [p for p in compute_params if p.requires_grad] if compute_params else [] + ctx.save_for_backward(x) + + # x.shape could be [bs, seqlen, hidden_size] or [seqlen, hidden_size] (moe experts) + x_shards = list(torch.chunk(x, chunks=shards, dim=-2)) + with torch.no_grad(): + output_shards = [fn(self, x_shard) for x_shard in x_shards] + output_unsharded = torch.cat(output_shards, dim=-2) + + return output_unsharded + + @staticmethod + def backward(ctx, *grads): + fn = ctx.fn + (x,) = ctx.saved_tensors + self = ctx.self + shards = ctx.shards + compute_params = ctx.compute_params + + x_requires_grad = x.requires_grad + x = x.detach() + # detach() unsets x.requires_grad, so restore it + x.requires_grad_(x_requires_grad) + + # x.shape could be [bs, seqlen, hidden_size] or [seqlen, hidden_size] (moe experts) + hidden_size = x.shape[-1] + x_shape_orig = x.shape + + # flatten bs+seqlen to avoid having stride issues when narrowing into seqlen w/ bs>1 + x = x.view(-1, hidden_size) + incoming_grad = grads[0].view(-1, hidden_size) + x_grad = torch.zeros_like(x) + + x_shards = list(torch.chunk(x, chunks=shards, dim=0)) + + for i, x_shard in enumerate(x_shards): + # Tell deepspeed not to add a new grad to its ipg bucket until the last shard is run + # XXX: DDP, FSDP will need something similar to make it work + if compute_params: + if i + 1 < shards: + for param in compute_params: + if hasattr(param, "ds_grad_is_ready"): + param.ds_grad_is_ready = False + else: + # last shard, can add the grad + for param in compute_params: + if hasattr(param, "ds_grad_is_ready"): + param.ds_grad_is_ready = True + + x_shard.requires_grad_(x_requires_grad) + + # if seqlen is not exactly divisible by shards the last step will be shorter than shard_step + shard_step = x_shards[i].shape[0] + shard_offset = i * x_shards[0].shape[0] + + x_shard.grad = x_grad.narrow(0, shard_offset, shard_step).view_as(x_shard) + incoming_grad_shard = incoming_grad.narrow(0, shard_offset, shard_step).view_as(x_shard) + with torch.enable_grad(): + output = fn(self, x_shard) + torch.autograd.backward(output, incoming_grad_shard) + + # unflatten + x_grad = x_grad.view(x_shape_orig) + + return (None, None, x_grad, None, None) + + +# DeepSpeed TiledMLP wrapper to match our interface +class DeepSpeedTiledMLPWrapper(nn.Module): + """ + Wrapper for DeepSpeed's TiledMLP to match the interface used in benchmarks. + Uses the DeepSpeed TiledMLP algorithm for memory-efficient MLP computation. + """ + + def __init__(self, config, num_shards=None): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.num_shards = num_shards + + self.mlp = LlamaMLP(config=config) + + def forward(self, x): + # Calculate num_shards if not provided + num_shards = self.num_shards + if num_shards is None: + hidden_size = x.shape[-1] + seqlen = x.shape[-2] + num_shards = math.ceil(seqlen / hidden_size) + num_shards = max(1, num_shards) + + # Collect compute parameters for DeepSpeed ZeRO compatibility + compute_params = [ + self.mlp.down_proj.weight, + self.mlp.gate_proj.weight, + self.mlp.up_proj.weight, + ] + + # Define the MLP forward function for DeepSpeed TiledMLP + def mlp_forward(mlp_module, x_input): + return mlp_module.down_proj(mlp_module.act_fn(mlp_module.gate_proj(x_input)) * mlp_module.up_proj(x_input)) + + # Use DeepSpeed's TiledMLP implementation + return DeepSpeedTiledMLP.apply( + mlp_forward, + self.mlp, + x, + num_shards, + compute_params, + ) + + +def bench_speed_tiled_mlp(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + seq_len = input.x + bsz = input.extra_benchmark_config["bsz"] + hidden_size = input.extra_benchmark_config["hidden_size"] + intermediate_size = input.extra_benchmark_config["intermediate_size"] + hidden_act = input.extra_benchmark_config["hidden_act"] + dtype = input.extra_benchmark_config["dtype"] + num_shards = input.extra_benchmark_config.get("num_shards", None) + activation_type = input.extra_benchmark_config["activation_type"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + llama_config = LlamaConfig( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + ) + + x_shape = (bsz, seq_len, hidden_size) + + # initialize input + x = torch.randn(*x_shape, device=device, dtype=dtype, requires_grad=True) + + if activation_type == "geglu": + if provider == "huggingface": + layer = LlamaMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger": + layer = LigerGEGLUMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger_tiled": + layer = LigerTiledGEGLUMLP(config=llama_config, num_shards=num_shards).to(device).to(dtype) + elif provider == "deepspeed_tiled": + layer = DeepSpeedTiledMLPWrapper(config=llama_config, num_shards=num_shards).to(device).to(dtype) + else: + raise ValueError(f"Invalid provider: {provider} for GEGLU") + elif activation_type == "swiglu": + if provider == "huggingface": + layer = LlamaMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger": + layer = LigerSwiGLUMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger_tiled": + layer = LigerTiledSwiGLUMLP(config=llama_config, num_shards=num_shards).to(device).to(dtype) + elif provider == "deepspeed_tiled": + layer = DeepSpeedTiledMLPWrapper(config=llama_config, num_shards=num_shards).to(device).to(dtype) + else: + raise ValueError(f"Invalid provider: {provider} for SwiGLU") + else: + raise ValueError(f"Invalid activation_type: {activation_type}") + + def fwd(): + return layer(x) + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd, + grad_to_none=[x], + rep=10, + quantiles=QUANTILES, + ) + elif mode == "backward": + do = torch.randn_like(x) + y = fwd() + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(do, retain_graph=True), + grad_to_none=[x], + rep=10, + quantiles=QUANTILES, + ) + else: + + def full(): + y = fwd() + y.backward(torch.randn_like(y), retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=[x], + rep=10, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_tiled_mlp(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + seq_len = input.x + bsz = input.extra_benchmark_config["bsz"] + hidden_size = input.extra_benchmark_config["hidden_size"] + intermediate_size = input.extra_benchmark_config["intermediate_size"] + hidden_act = input.extra_benchmark_config["hidden_act"] + dtype = input.extra_benchmark_config["dtype"] + num_shards = input.extra_benchmark_config.get("num_shards", None) + activation_type = input.extra_benchmark_config["activation_type"] + provider = input.kernel_provider + mode = input.kernel_operation_mode + + llama_config = LlamaConfig( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + ) + + x_shape = (bsz, seq_len, hidden_size) + # initialize input + x = torch.randn(*x_shape, device=device, dtype=dtype, requires_grad=True) + + if activation_type == "geglu": + if provider == "huggingface": + layer = LlamaMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger": + layer = LigerGEGLUMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger_tiled": + layer = LigerTiledGEGLUMLP(config=llama_config, num_shards=num_shards).to(device).to(dtype) + elif provider == "deepspeed_tiled": + layer = DeepSpeedTiledMLPWrapper(config=llama_config, num_shards=num_shards).to(device).to(dtype) + else: + raise ValueError(f"Invalid provider: {provider} for GEGLU") + elif activation_type == "swiglu": + if provider == "huggingface": + layer = LlamaMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger": + layer = LigerSwiGLUMLP(config=llama_config).to(device).to(dtype) + elif provider == "liger_tiled": + layer = LigerTiledSwiGLUMLP(config=llama_config, num_shards=num_shards).to(device).to(dtype) + elif provider == "deepspeed_tiled": + layer = DeepSpeedTiledMLPWrapper(config=llama_config, num_shards=num_shards).to(device).to(dtype) + else: + raise ValueError(f"Invalid provider: {provider} for SwiGLU") + else: + raise ValueError(f"Invalid activation_type: {activation_type}") + + def fwd(): + return layer(x) + + def full(): + y = fwd() + y.backward(torch.randn_like(y), retain_graph=True) + + if mode == "forward": + mem_50, mem_20, mem_80 = _test_memory( + fwd, + quantiles=QUANTILES, + ) + elif mode == "backward": + do = torch.randn_like(x) + y = fwd() + mem_50, mem_20, mem_80 = _test_memory( + lambda: y.backward(do, retain_graph=True), + quantiles=QUANTILES, + ) + else: + mem_50, mem_20, mem_80 = _test_memory( + full, + quantiles=QUANTILES, + ) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + + # Benchmark GEGLU variants + kernel_providers_geglu = ["huggingface", "liger", "liger_tiled", "deepspeed_tiled"] + + common_configs_geglu = { + "kernel_name": "tiled_geglu", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, 15)], # 1024 to 16384 + "kernel_providers": kernel_providers_geglu, + "extra_benchmark_configs": [ + { + "bsz": 2, + "hidden_size": 2048, + "intermediate_size": 4096, + "hidden_act": "gelu_pytorch_tanh", + "activation_type": "geglu", + "num_shards": 4, + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_tiled_mlp, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs_geglu, + ) + run_benchmarks( + bench_test_fn=bench_memory_tiled_mlp, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="memory", + metric_unit="MB", + **common_configs_geglu, + ) + + # Benchmark SwiGLU variants + kernel_providers_swiglu = ["huggingface", "liger", "liger_tiled", "deepspeed_tiled"] + + common_configs_swiglu = { + "kernel_name": "tiled_swiglu", + "x_name": "T", + "x_label": "sequence length", + "x_values": [2**i for i in range(10, 15)], # 1024 to 16384 + "kernel_providers": kernel_providers_swiglu, + "extra_benchmark_configs": [ + { + "bsz": 2, + "hidden_size": 2048, + "intermediate_size": 4096, + "hidden_act": "silu", + "activation_type": "swiglu", + "num_shards": 4, + "dtype": torch.bfloat16, + } + ], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_speed_tiled_mlp, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="speed", + metric_unit="ms", + **common_configs_swiglu, + ) + run_benchmarks( + bench_test_fn=bench_memory_tiled_mlp, + kernel_operation_modes=["full", "forward", "backward"], + metric_name="memory", + metric_unit="MB", + **common_configs_swiglu, + ) diff --git a/benchmark/scripts/benchmark_tvd.py b/benchmark/scripts/benchmark_tvd.py new file mode 100755 index 0000000..ef76380 --- /dev/null +++ b/benchmark/scripts/benchmark_tvd.py @@ -0,0 +1,145 @@ +import torch +import triton + +from utils import QUANTILES +from utils import SingleBenchmarkRunInput +from utils import SingleBenchmarkRunOutput +from utils import _test_memory +from utils import parse_benchmark_script_args +from utils import run_benchmarks + +from liger_kernel.transformers.tvd import LigerTVDLoss +from liger_kernel.utils import get_total_gpu_memory +from liger_kernel.utils import infer_device + +device = infer_device() + + +class TorchTVDLoss(torch.nn.Module): + def __init__(self, reduction="batchmean"): + super(TorchTVDLoss, self).__init__() + self.reduction = reduction + + def forward(self, p, q): + tvd = torch.abs(p - q) / 2.0 + if self.reduction == "mean": + return torch.sum(tvd) / (p.size(0) * p.size(1)) + elif self.reduction == "sum": + return torch.sum(tvd) + elif self.reduction == "none": + return tvd + elif self.reduction == "batchmean": + return torch.sum(tvd) / p.size(0) + else: + raise ValueError("Invalid reduction type.") + + +S, E = 12, 18 + + +def bench_speed_tvd(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + reduction = "batchmean" + V = input.x + B, T = input.extra_benchmark_config["B"], input.extra_benchmark_config["T"] + torch_tvd = TorchTVDLoss(reduction=reduction) + liger_tvd = LigerTVDLoss(reduction=reduction) + + _input = torch.randn(B * T, V, requires_grad=True, device=device).softmax(dim=-1) + target = torch.randn(B * T, V, device=device).softmax(dim=-1) + + def fwd(): + if input.kernel_provider == "liger": + return liger_tvd(_input, target) + else: + return torch_tvd(_input, target) + + if input.kernel_operation_mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench(fwd, quantiles=QUANTILES, rep=100) + elif input.kernel_operation_mode == "backward": + y = fwd() + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(retain_graph=True), + quantiles=QUANTILES, + grad_to_none=[_input], + rep=100, + ) + elif input.kernel_operation_mode == "full": + + def full(): + y = fwd() + y.backward(retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench(full, quantiles=QUANTILES, rep=100) + return SingleBenchmarkRunOutput( + y_20=ms_20, + y_50=ms_50, + y_80=ms_80, + ) + + +def bench_memory_tvd(input: SingleBenchmarkRunInput) -> SingleBenchmarkRunOutput: + reduction = "batchmean" + torch_tvd = TorchTVDLoss(reduction=reduction) + liger_tvd = LigerTVDLoss(reduction=reduction) + + V = input.x + B, T = input.extra_benchmark_config["B"], input.extra_benchmark_config["T"] + + _input = torch.randn(B * T, V, requires_grad=True, device=device).softmax(dim=-1) + target = torch.randn(B * T, V, device=device).softmax(dim=-1) + + def fwd(): + if input.kernel_provider == "liger": + return liger_tvd(_input, target) + else: + return torch_tvd(_input, target) + + def full(): + y = fwd() + y.backward(retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + + return SingleBenchmarkRunOutput( + y_20=mem_20, + y_50=mem_50, + y_80=mem_80, + ) + + +if __name__ == "__main__": + args = parse_benchmark_script_args() + gpu_memory_gbs = get_total_gpu_memory() + # We know that the full test will require 66GBs for vocab size 2^17 + if gpu_memory_gbs >= 66: + x_max = 17 + elif gpu_memory_gbs >= 32: + x_max = 16 + else: + x_max = 15 + common_args = { + "kernel_name": "tvd", + "x_name": "V", + "x_label": "vocab size", + "x_values": [2**i for i in range(12, x_max + 1)], + "kernel_providers": ["liger", "torch"], + "extra_benchmark_configs": [{"B": 8, "T": 2048}], + "overwrite": args.overwrite, + } + + run_benchmarks( + bench_test_fn=bench_memory_tvd, + kernel_operation_modes=["full"], + metric_name="memory", + metric_unit="MB", + **common_args, + ) + + run_benchmarks( + bench_test_fn=bench_speed_tvd, + kernel_operation_modes=["forward", "full", "backward"], + metric_name="speed", + metric_unit="ms", + **common_args, + ) diff --git a/benchmark/scripts/utils.py b/benchmark/scripts/utils.py new file mode 100755 index 0000000..e6b4fc9 --- /dev/null +++ b/benchmark/scripts/utils.py @@ -0,0 +1,439 @@ +import argparse +import csv +import json +import os +import time + +from collections import OrderedDict +from dataclasses import asdict +from dataclasses import dataclass +from importlib.metadata import version +from itertools import zip_longest +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Optional +from typing import Union + +import torch + +from liger_kernel.utils import infer_device + +device = infer_device() + +LIGER_KERNEL_VERSION = version("liger-kernel") + +QUANTILES = [0.5, 0.2, 0.8] + + +@dataclass +class SingleBenchmarkRunInput: + x: Union[int, float] + kernel_provider: str + kernel_operation_mode: Optional[str] = "" + extra_benchmark_config: Optional[Dict[str, Any]] = None + + +@dataclass +class SingleBenchmarkRunOutput: + # 20th percentile + y_20: float + # 50th percentile (median) + y_50: float + # 80th percentile + y_80: float + + +@dataclass +class BenchmarkData: + """ + BenchmarkData is a dataclass to store the benchmark data for a a completed benchmark + run on all x-values for a given kernel/kernel operation mode/metric/extra_benchmark_config + """ + + kernel_name: str + kernel_provider: str + metric_name: str + metric_unit: str + gpu_name: str + x_name: str + x_label: str + x_values: List[float] + y_values_50: List[float] + y_values_20: List[float] + y_values_80: List[float] + timestamp: str + kernel_operation_mode: Optional[str] = None + extra_benchmark_config_str: Optional[str] = None + liger_version: str = LIGER_KERNEL_VERSION + + +@dataclass +class BenchmarkDataCSVRow: + # The ordering of field names here will be the order of columns in the CSV + kernel_name: str + kernel_provider: str + kernel_operation_mode: Union[str, None] + metric_name: str + metric_unit: str + x_name: str + x_label: str + x_value: float + y_value_50: float + y_value_20: float + y_value_80: float + extra_benchmark_config_str: Union[str, None] + gpu_name: str + timestamp: str + liger_version: str + + +def _test_memory( + func: Callable, + _iter: int = 10, + quantiles: Optional[List[float]] = None, + return_mode="mean", +) -> float: + assert return_mode in ["min", "max", "mean", "median"] + total_mem = [] + + for _ in range(_iter): + getattr(torch, device).memory.reset_peak_memory_stats() + func() + # Convert to MB + mem = getattr(torch, device).max_memory_allocated() / 2**20 + total_mem.append(mem) + + total_mem = torch.tensor(total_mem, dtype=torch.float) + if quantiles is not None: + quantiles_data = torch.quantile(total_mem, torch.tensor(quantiles, dtype=torch.float)).tolist() + if len(quantiles_data) == 1: + quantiles_data = quantiles_data[0] + return quantiles_data + return getattr(torch, return_mode)(total_mem).item() + + +def run_speed_benchmark( + fwd_fn: Callable, + mode: str, + input_tensors: List[torch.Tensor], + rep: int = 10, +) -> "SingleBenchmarkRunOutput": + """Measure execution speed for forward, backward, or full (fwd+bwd). + + Covers the common case where the forward function returns a single tensor + and backward uses a random gradient of the same shape. For kernels with + scalar output (losses) or multiple outputs (e.g. RoPE), write custom + measurement logic instead. + """ + import triton + + if mode == "forward": + ms_50, ms_20, ms_80 = triton.testing.do_bench( + fwd_fn, + grad_to_none=input_tensors, + rep=rep, + quantiles=QUANTILES, + ) + elif mode == "backward": + y = fwd_fn() + do = torch.randn_like(y) + ms_50, ms_20, ms_80 = triton.testing.do_bench( + lambda: y.backward(do, retain_graph=True), + grad_to_none=input_tensors, + rep=rep, + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd_fn() + y.backward(torch.randn_like(y), retain_graph=True) + + ms_50, ms_20, ms_80 = triton.testing.do_bench( + full, + grad_to_none=input_tensors, + rep=rep, + quantiles=QUANTILES, + ) + else: + raise ValueError(f"Unsupported mode: {mode}. Use 'forward', 'backward', or 'full'.") + return SingleBenchmarkRunOutput(y_20=ms_20, y_50=ms_50, y_80=ms_80) + + +def run_memory_benchmark( + fwd_fn: Callable, + mode: str, +) -> "SingleBenchmarkRunOutput": + """Measure peak memory for forward, backward, or full (fwd+bwd). + + Same caveats as :func:`run_speed_benchmark` regarding output shape. + """ + if mode == "forward": + mem_50, mem_20, mem_80 = _test_memory(fwd_fn, quantiles=QUANTILES) + elif mode == "backward": + y = fwd_fn() + do = torch.randn_like(y) + mem_50, mem_20, mem_80 = _test_memory( + lambda: y.backward(do, retain_graph=True), + quantiles=QUANTILES, + ) + elif mode == "full": + + def full(): + y = fwd_fn() + y.backward(torch.randn_like(y), retain_graph=True) + + mem_50, mem_20, mem_80 = _test_memory(full, quantiles=QUANTILES) + else: + raise ValueError(f"Unsupported mode: {mode}. Use 'forward', 'backward', or 'full'.") + return SingleBenchmarkRunOutput(y_20=mem_20, y_50=mem_50, y_80=mem_80) + + +def get_current_file_directory() -> str: + """ + Returns the directory path of the current Python file. + """ + # Get the absolute path of the current file + current_file_path = os.path.abspath(__file__) + + # Get the directory path of the current file + return os.path.dirname(current_file_path) + + +def sleep(seconds): + def decorator(function): + def wrapper(*args, **kwargs): + time.sleep(seconds) + return function(*args, **kwargs) + + return wrapper + + return decorator + + +def _print_benchmarking_banner(metric_name: str, kernel_name: str): + print("**************************************") + print(f" BENCHMARKING {metric_name.upper()} for {kernel_name.upper()}") + print("**************************************") + + +def get_formatted_time(): + return time.strftime("%Y-%m-%d %H:%M:%S") + + +def get_gpu_name(): + """ + Returns the current GPU name, formatted to serve as a directory name + """ + torch_device = getattr(torch, device) + if torch_device.is_available(): + gpu_name = torch_device.get_device_name(torch_device.current_device()) + return gpu_name + else: + raise Exception("Benchmarks can only be run on GPU.") + + +def update_benchmark_data_csv( + benchmark_data_list: List[BenchmarkData], + filename: str = "all_benchmark_data.csv", + overwrite: bool = True, +): + """ + Update the CSV file with the new benchmark data. If the file does not exist, create it. + If an entry already exists for the benchmark, then overwrite it if `overwrite` is True. + """ + + def create_unique_key(row): + # This unique key is used to determine if a benchmark run already exists in the CSV + # If the key is the same, then the benchmark run already exists and will optionally + # be overwritten. Otherwise, it is considered a new benchmark run and appended. + return ( + row["kernel_name"], + row["kernel_provider"], + row["kernel_operation_mode"] if row["kernel_operation_mode"] else "", + row["metric_name"], + row["x_name"], + str(row["x_value"]), + (row["extra_benchmark_config_str"] if row["extra_benchmark_config_str"] else ""), + row["gpu_name"], + ) + + fieldnames = BenchmarkDataCSVRow.__annotations__.keys() + + # Make filename path relative to current file + filename_abs_path = os.path.join(get_current_file_directory(), "../data", filename) + file_exists = os.path.isfile(filename_abs_path) + + # Read existing data into a list of dicts + existing_data = [] + if file_exists: + with open(filename_abs_path, mode="r") as file: + reader = csv.DictReader(file) + for row in reader: + existing_data.append(row) + + existing_data_dict = OrderedDict((create_unique_key(row), row) for row in existing_data) + + for benchmark_data in benchmark_data_list: + benchmark_data_dict = asdict(benchmark_data) + x_values = benchmark_data_dict.pop("x_values") + y_values_50 = benchmark_data_dict.pop("y_values_50") + y_values_20 = benchmark_data_dict.pop("y_values_20") + y_values_80 = benchmark_data_dict.pop("y_values_80") + + # Need to convert benchmark_data into multiple rows based on x_values and y_values + for x_value, y_value_50, y_value_20, y_value_80 in zip_longest(x_values, y_values_50, y_values_20, y_values_80): + if y_value_50 is None: + y_value_50 = float("nan") + if y_value_20 is None: + y_value_20 = float("nan") + if y_value_80 is None: + y_value_80 = float("nan") + + row = BenchmarkDataCSVRow( + x_value=x_value, + y_value_50=y_value_50, + y_value_20=y_value_20, + y_value_80=y_value_80, + **benchmark_data_dict, + ) + row_dict = asdict(row) + + row_key = create_unique_key(row_dict) + + if row_key in existing_data_dict: + if overwrite: + # If overwriting, update the row + existing_data_dict[row_key] = row_dict + else: + # If not overwriting, skip this row + pass + else: + existing_data_dict[row_key] = row_dict + os.makedirs(os.path.dirname(filename_abs_path), exist_ok=True) + with open(filename_abs_path, mode="w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=fieldnames) + writer.writeheader() + + for row in existing_data_dict.values(): + writer.writerow(row) + + +class CustomEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, torch.dtype): + return str(obj) + return super().default(self, obj) + + +def print_benchmark_data(benchmark_data_list: List[BenchmarkData]) -> str: + print("********** Benchmark Data **********") + formatted_list = [obj.__dict__ for obj in benchmark_data_list] + print(json.dumps(formatted_list, indent=2)) + + +def run_benchmarks( + bench_test_fn: Callable, + kernel_name: str, + metric_name: str, + metric_unit: str, + x_name: str, + x_label: str, + x_values: List[Union[float, int]], + kernel_providers: List[str], + kernel_operation_modes: Optional[List[str]] = [None], + extra_benchmark_configs: Optional[List[Dict[str, Any]]] = None, + overwrite: bool = False, +): + """ + Run benchmarks given a bench_test_fn that takes in a SingleBenchmarkRunInput as input and + saves data to the CSV file. + + Args: + - bench_test_fn: The benchmark test function to run. This function should take in a + SingleBenchmarkRunInput as input and return a SingleBenchmarkRunOutput. + - kernel_name: The name of the kernel being benchmarked (e.g. "swiglu") + - metric_name: The name of the metric being benchmarked (e.g. "speed" or "memory") + - metric_unit: The unit of the metric being benchmarked (e.g. "ms" or "MB") + - x_name: The name of the x-axis (e.g. "T" for sequence length) + - x_label: The label of the x-axis (e.g. "sequence length") + - x_values: The list of x-values to run the benchmark on (e.g. [2**i for i in range(10, 14)]) + - kernel_providers: The list of kernel providers to run the benchmark on (e.g. ["liger", "huggingface"]) + - kernel_operation_modes: The list of kernel operation modes to run the benchmark on (e.g. ["full", "backward"]) + - extra_benchmark_configs: The list of extra benchmark configurations to run the benchmark on. + - overwrite: Whether to overwrite the existing benchmark data entry if it already exists. + """ + + assert len(kernel_operation_modes) >= 1 + assert len(kernel_providers) >= 1 + + _print_benchmarking_banner(metric_name=metric_name, kernel_name=kernel_name) + + gpu_name = get_gpu_name() + benchmark_data_list = [] + for extra_benchmark_config in extra_benchmark_configs: + for kernel_operation_mode in kernel_operation_modes: + for kernel_provider in kernel_providers: + y_values_50 = [] + y_values_20 = [] + y_values_80 = [] + + for x in x_values: + single_benchmark_run_input = SingleBenchmarkRunInput( + x=x, + kernel_provider=kernel_provider, + kernel_operation_mode=kernel_operation_mode, + extra_benchmark_config=extra_benchmark_config, + ) + benchmark_result: SingleBenchmarkRunOutput = bench_test_fn(single_benchmark_run_input) + y_values_50.append(benchmark_result.y_50) + y_values_20.append(benchmark_result.y_20) + y_values_80.append(benchmark_result.y_80) + + benchmark_run_data = BenchmarkData( + kernel_name=kernel_name, + kernel_operation_mode=kernel_operation_mode, + kernel_provider=kernel_provider, + metric_name=metric_name, + metric_unit=metric_unit, + gpu_name=gpu_name, + x_name=x_name, + x_label=x_label, + x_values=x_values, + y_values_50=y_values_50, + y_values_20=y_values_20, + y_values_80=y_values_80, + extra_benchmark_config_str=json.dumps(extra_benchmark_config, cls=CustomEncoder), + timestamp=get_formatted_time(), + liger_version=LIGER_KERNEL_VERSION, + ) + + benchmark_data_list.append(benchmark_run_data) + + print_benchmark_data(benchmark_data_list) + + update_benchmark_data_csv(benchmark_data_list=benchmark_data_list, overwrite=overwrite) + + +def parse_benchmark_script_args(): + parser = argparse.ArgumentParser(description="Benchmarking script for Liger-Kernel") + + parser.add_argument( + "--overwrite", + action="store_true", + help="Flag to overwrite existing benchmark data with current run.", + ) + parser.add_argument( + "--model", + type=str, + default=None, + help=( + "Model config name from MODEL_REGISTRY " + "(e.g. llama_2_7b, llama_3_8b). " + "Defaults to llama_3_8b when not specified." + ), + ) + args = parser.parse_args() + return args diff --git a/dev/fmt-requirements.txt b/dev/fmt-requirements.txt new file mode 100755 index 0000000..1d8f486 --- /dev/null +++ b/dev/fmt-requirements.txt @@ -0,0 +1 @@ +ruff>=0.1.6 diff --git a/dev/modal/benchmarks.py b/dev/modal/benchmarks.py new file mode 100755 index 0000000..a54fa47 --- /dev/null +++ b/dev/modal/benchmarks.py @@ -0,0 +1,73 @@ +from pathlib import Path + +import modal + +ROOT_PATH = Path(__file__).parent.parent.parent +REMOTE_ROOT_PATH = "/root/liger-kernel" +PYTHON_VERSION = "3.12" + +image = modal.Image.debian_slim(python_version=PYTHON_VERSION).pip_install("uv") + +app = modal.App("liger_benchmarks", image=image) + +# mount: add local files to the remote container +repo = image.add_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH) + + +@app.function(gpu="H100!", image=repo, timeout=60 * 90) +def liger_benchmarks(): + import os + import subprocess + + subprocess.run( + ["uv pip install -e '.[dev]' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run(["make run-benchmarks"], check=True, shell=True, cwd=REMOTE_ROOT_PATH) + + file_path = Path(REMOTE_ROOT_PATH) / "benchmark" / "data" / "all_benchmark_data.csv" + print(f"Checking if file exists at: {file_path}") + print(f"File exists: {os.path.exists(file_path)}") + + if not os.path.exists(file_path): + print("Listing directory contents:") + data_dir = file_path.parent + if os.path.exists(data_dir): + print(f"Contents of {data_dir}:") + print(os.listdir(data_dir)) + else: + print(f"Data directory {data_dir} does not exist") + raise FileNotFoundError(f"Benchmark data file not found at {file_path}") + + with open(file_path, "rb") as f: + data = f.read() + print(f"Successfully read {len(data)} bytes of data") + return data + + +@app.local_entrypoint() +def main(): + try: + # Run the benchmarks and get the data + print("Starting benchmark run...") + benchmark_data = liger_benchmarks.remote() + + if not benchmark_data: + raise ValueError("No data received from remote function") + + # Save the data locally + local_data_path = ROOT_PATH / "benchmark" / "data" / "all_benchmark_data.csv" + print(f"Attempting to save data to: {local_data_path}") + + local_data_path.parent.mkdir(parents=True, exist_ok=True) + + with open(local_data_path, "wb") as f: + f.write(benchmark_data) + + print(f"Successfully saved {len(benchmark_data)} bytes to: {local_data_path}") + + except Exception as e: + print(f"Error occurred: {str(e)}") + raise diff --git a/dev/modal/tests.py b/dev/modal/tests.py new file mode 100755 index 0000000..07856dc --- /dev/null +++ b/dev/modal/tests.py @@ -0,0 +1,86 @@ +from pathlib import Path + +import modal + +ROOT_PATH = Path(__file__).parent.parent.parent +REMOTE_ROOT_PATH = "/root/liger-kernel" +PYTHON_VERSION = "3.12" + +OLDEST_SUPPORTED_TRANSFORMERS_V4_VERSION = "4.52.0" + +image = modal.Image.debian_slim(python_version=PYTHON_VERSION).pip_install("uv") + +app = modal.App("liger_tests", image=image) + +# mount: add local files to the remote container +repo = image.add_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH) + + +@app.function(gpu="H100!", image=repo, timeout=90 * 60) +def liger_correctness_tests(): + import subprocess + + subprocess.run( + ["uv pip install -e '.[dev]' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run(["make test"], check=True, shell=True, cwd=REMOTE_ROOT_PATH) + + +@app.function(gpu="H100!", image=repo, timeout=90 * 60) +def liger_convergence_tests(): + import subprocess + + subprocess.run( + ["uv pip install -e '.[dev]' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run(["make test-convergence"], check=True, shell=True, cwd=REMOTE_ROOT_PATH) + + +oldest_v4_app = modal.App("liger_oldest_v4_tests", image=image) # 4.52.0 + + +@oldest_v4_app.function(gpu="H100!", image=repo, timeout=90 * 60) +def liger_oldest_v4_correctness_tests(): + import subprocess + + subprocess.run( + ["uv pip install -e '.[dev]' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run( + [f"uv pip install 'transformers=={OLDEST_SUPPORTED_TRANSFORMERS_V4_VERSION}' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run(["make test"], check=True, shell=True, cwd=REMOTE_ROOT_PATH) + + +@oldest_v4_app.function(gpu="H100!", image=repo, timeout=90 * 60) +def liger_oldest_v4_convergence_tests(): + import subprocess + + subprocess.run( + ["uv pip install -e '.[dev]' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run( + [f"uv pip install 'transformers=={OLDEST_SUPPORTED_TRANSFORMERS_V4_VERSION}' --system"], + check=True, + shell=True, + cwd=REMOTE_ROOT_PATH, + ) + subprocess.run(["make test-convergence"], check=True, shell=True, cwd=REMOTE_ROOT_PATH) + + +latest_v4_app = modal.App("liger_latest_v4_tests", image=image) # 4.57.6 diff --git a/docs/Examples.md b/docs/Examples.md new file mode 100755 index 0000000..41a1fb9 --- /dev/null +++ b/docs/Examples.md @@ -0,0 +1,268 @@ + +!!! Example "HANDS-ON USECASE EXAMPLES" +| **Use Case** | **Description** | +|------------------------------------------------|---------------------------------------------------------------------------------------------------| +| [**Hugging Face Trainer**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/huggingface) | Train LLaMA 3-8B ~20% faster with over 40% memory reduction on Alpaca dataset using 4 A100s with FSDP | +| [**Lightning Trainer**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/lightning) | Increase 15% throughput and reduce memory usage by 40% with LLaMA3-8B on MMLU dataset using 8 A100s with DeepSpeed ZeRO3 | +| [**Medusa Multi-head LLM (Retraining Phase)**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/medusa) | Reduce memory usage by 80% with 5 LM heads and improve throughput by 40% using 8 A100s with FSDP | +| [**Vision-Language Model SFT**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/huggingface/run_qwen2_vl.sh) | Finetune Qwen2-VL on image-text data using 4 A100s with FSDP | +| [**Liger ORPO Trainer**](https://github.com/linkedin/Liger-Kernel/blob/main/examples/alignment/run_orpo.py) | Align Llama 3.2 using Liger ORPO Trainer with FSDP with 50% memory reduction | + +## HuggingFace Trainer + +### How to Run + +#### Locally on a GPU machine +You can run the example locally on a GPU machine. The default hyperparameters and configurations work on single node with 4xA100 80GB GPUs and FSDP. + +!!! Example + +```bash +pip install -r requirements.txt +sh run_{MODEL}.sh +``` + +#### Remotely on Modal +If you do not have access to a GPU machine, you can run the example on Modal. Modal is a serverless platform that allows you to run your code on a remote GPU machine. You can sign up for a free account at [Modal](https://www.modal.com/). + +!!! Example + +```bash +pip install modal +modal setup # authenticate with Modal +modal run launch_on_modal.py --script "run_qwen2_vl.sh" +``` + +!!! Notes + +1. This example uses an optional `use_liger` flag. If true, it does a 1 line monkey patch to apply liger kernel. + +2. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the following: + * Agree on the [community license agreement](https://huggingface.co/meta-llama/Meta-Llama-3-8B) . + * Run `huggingface-cli login` and enter your HuggingFace token. + +3. The default hyperparameters and configurations work on single node with 4xA100 80GB GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. + + +### Benchmark Result + +### Llama + +!!! Info +>Benchmark conditions: +>Model= LLaMA 3-8B,Datset= Alpaca, Max seq len = 512, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 4 A100s. + +Throughput improves by around 20%, while GPU memory usage drops by 40%. This allows you to train the model on smaller GPUs, use larger batch sizes, or handle longer sequence lengths without incurring additional costs. + +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/huggingface/img/llama_tps.png) +![GPU Memory Allocated](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/huggingface/img/llama_mem_alloc.png) + +### Qwen + +!!! Info +>Benchmark conditions: +>Model= Qwen2-7B, Dataset= Alpaca, Max seq len = 512, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 4 A100s. + +Throughput improves by around 10%, while GPU memory usage drops by 50%. + +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/huggingface/img/qwen_tps.png) +![GPU Memory Allocated](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/huggingface/img/qwen_mem_alloc.png) + + +### Gemma 7B + +!!! Info +>Benchmark conditions: +> Model= Gemma-7B, Dataset= Alpaca, Max seq len = 512, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 4 A100s. + +Throughput improves by around 24%, while GPU memory usage drops by 33%. + +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/huggingface/img/gemma_7b_mem.png) +![GPU Memory Allocated](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/huggingface/img/gemma_7b_tp.png) + +## Lightning Trainer + +### How to Run + +#### Locally on a GPU machine +You can run the example locally on a GPU machine. + +!!! Example + +```bash +pip install -r requirements.txt + +# For single L40 48GB GPU +python training.py --model Qwen/Qwen2-0.5B-Instruct --num_gpu 1 --max_length 1024 + +# For 8XA100 40GB +python training.py --model meta-llama/Meta-Llama-3-8B --strategy deepspeed +``` + +!!! Notes + +1. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the following: + * Agree on the [community license agreement](https://huggingface.co/meta-llama/Meta-Llama-3-8B) + * Run `huggingface-cli login` and enter your HuggingFace token. + +2. The default hyperparameters and configurations for gemma works on single L40 48GB GPU and config for llama work on single node with 8xA100 40GB GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. + +## Medusa + +Medusa is a simple framework that democratizes the acceleration techniques for LLM generation with multiple decoding heads. To know more, you can check out the [repo](https://arxiv.org/abs/2401.10774) and the [paper](https://arxiv.org/abs/2401.10774) . + +The Liger fused CE kernel is highly effective in this scenario, eliminating the need to materialize logits for each head, which usually consumes a large volume of memory due to the extensive vocabulary size (e.g., for LLaMA-3, the vocabulary size is 128k). + +The introduction of multiple heads can easily lead to OOM (Out of Memory) issues. However, thanks to the efficient Liger fused CE, which calculates the gradient in place and doesn't materialize the logits, we have observed very effective results. This efficiency opens up more opportunities for multi-token prediction research and development. + + +### How to Run + +!!! Example + +```bash +git clone git@github.com:linkedin/Liger-Kernel.git +cd {PATH_TO_Liger-Kernel}/Liger-Kernel/ +pip install -e . +cd {PATH_TO_Liger-Kernel}/Liger-Kernel/examples/medusa +pip install -r requirements.txt +sh scripts/llama3_8b_medusa.sh +``` + +!!! Notes + +1. This example uses an optional `use_liger` flag. If true, it does a monkey patch to apply liger kernel with medusa heads. + +2. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the followings: + * Agree on the community license agreement https://huggingface.co/meta-llama/Meta-Llama-3-8B + * Run `huggingface-cli login` and enter your HuggingFace token + +3. The default hyperparameters and configurations work on single node with 8xA100 GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. + +4. We are using a smaller sample of shared GPT data primarily to benchmark performance. The example requires hyperparameter tuning and dataset selection to work effectively, also ensuring the dataset has the same distribution as the LLaMA pretraining data. Welcome contribution to enhance the example code. + +### Benchmark Result + +!!! Info +> 1. Benchmark conditions: LLaMA 3-8B, Batch Size = 6, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 8 A100s. + +#### Stage 1 + +Stage 1 refers to Medusa-1 where the backbone model is frozen and only weights of LLM heads are updated. + +!!! Warning +```bash +# Modify this flag in llama3_8b_medusa.sh to True enables stage1 +--medusa_only_heads True +``` + +#### num_head = 3 + +![Memory](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Memory_Stage1_num_head_3.png) +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Throughput_Stage1_num_head_3.png) + +#### num_head = 5 + +![Memory](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Memory_Stage1_num_head_5.png) +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Throughput_Stage1_num_head_5.png) + +#### Stage 2 + +!!! Warning +```bash +# Modify this flag to False in llama3_8b_medusa.sh enables stage2 +--medusa_only_heads False +``` + +Stage 2 refers to Medusa-2 where all the model weights are updated including the backbone model and llm heads. + +#### num_head = 3 + +![Memory](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Memory_Stage2_num_head_3.png) +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Throughput_Stage2_num_head_3.png) + +#### num_head = 5 + +![Memory](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Memory_Stage2_num_head_5.png) +![Throughput](https://raw.githubusercontent.com/linkedin/Liger-Kernel/main/examples/medusa/docs/images/Throughput_Stage2_num_head_5.png) + + +## Vision-Language Model SFT + +## How to Run + +### Locally on a GPU Machine +You can run the example locally on a GPU machine. The default hyperparameters and configurations work on single node with 4xA100 80GB GPUs. + +!!! Example +```bash +#!/bin/bash + +torchrun --nnodes=1 --nproc-per-node=4 training_multimodal.py \ + --model_name "Qwen/Qwen2-VL-7B-Instruct" \ + --bf16 \ + --num_train_epochs 1 \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 6e-6 \ + --weight_decay 0.05 \ + --warmup_ratio 0.1 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --include_num_input_tokens_seen \ + --report_to none \ + --fsdp "full_shard auto_wrap" \ + --fsdp_config config/fsdp_config.json \ + --seed 42 \ + --use_liger True \ + --output_dir multimodal_finetuning +``` + +## ORPO Trainer + +### How to Run + +#### Locally on a GPU Machine + +You can run the example locally on a GPU machine and FSDP. + +!!! Example +```py +import torch +from datasets import load_dataset +from transformers import AutoModelForCausalLM, AutoTokenizer +from trl import ORPOConfig # noqa: F401 + +from liger_kernel.transformers.trainer import LigerORPOTrainer # noqa: F401 + +model = AutoModelForCausalLM.from_pretrained( + "meta-llama/Llama-3.2-1B-Instruct", + dtype=torch.bfloat16, +) + +tokenizer = AutoTokenizer.from_pretrained( + "meta-llama/Llama-3.2-1B-Instruct", + max_length=512, + padding="max_length", +) +tokenizer.pad_token = tokenizer.eos_token + +train_dataset = load_dataset("trl-lib/tldr-preference", split="train") + +training_args = ORPOConfig( + output_dir="Llama3.2_1B_Instruct", + beta=0.1, + max_length=128, + per_device_train_batch_size=32, + max_steps=100, + save_strategy="no", +) + +trainer = LigerORPOTrainer( + model=model, args=training_args, tokenizer=tokenizer, train_dataset=train_dataset +) + +trainer.train() +``` \ No newline at end of file diff --git a/docs/Getting-Started.md b/docs/Getting-Started.md new file mode 100755 index 0000000..3b6af54 --- /dev/null +++ b/docs/Getting-Started.md @@ -0,0 +1,64 @@ +There are a couple of ways to apply Liger kernels, depending on the level of customization required. + +### 1. Use AutoLigerKernelForCausalLM + +Using the `AutoLigerKernelForCausalLM` is the simplest approach, as you don't have to import a model-specific patching API. If the model type is supported, the modeling code will be automatically patched using the default settings. + +!!! Example + + ```python + from liger_kernel.transformers import AutoLigerKernelForCausalLM + + # This AutoModel wrapper class automatically monkey-patches the + # model with the optimized Liger kernels if the model is supported. + model = AutoLigerKernelForCausalLM.from_pretrained("path/to/some/model") + ``` + +### 2. Apply Model-Specific Patching APIs + +Using the [patching APIs](https://github.com/linkedin/Liger-Kernel?tab=readme-ov-file#patching), you can swap Hugging Face models with optimized Liger Kernels. + +!!! Example + +```python +import transformers +from liger_kernel.transformers import apply_liger_kernel_to_llama + +# 1a. Adding this line automatically monkey-patches the model with the optimized Liger kernels +apply_liger_kernel_to_llama() + +# 1b. You could alternatively specify exactly which kernels are applied +apply_liger_kernel_to_llama( + rope=True, + swiglu=True, + cross_entropy=True, + fused_linear_cross_entropy=False, + rms_norm=False +) + +# 2. Instantiate patched model +model = transformers.AutoModelForCausalLM("path/to/llama/model") +``` + +### 3. Compose Your Own Model + +You can take individual [kernels](https://github.com/linkedin/Liger-Kernel?tab=readme-ov-file#model-kernels) to compose your models. + +!!! Example + +```python +from liger_kernel.transformers import LigerFusedLinearCrossEntropyLoss +import torch.nn as nn +import torch + +model = nn.Linear(128, 256).cuda() + +# fuses linear + cross entropy layers together and performs chunk-by-chunk computation to reduce memory +loss_fn = LigerFusedLinearCrossEntropyLoss() + +input = torch.randn(4, 128, requires_grad=True, device="cuda") +target = torch.randint(256, (4, ), device="cuda") + +loss = loss_fn(model.weight, input, target) +loss.backward() +``` \ No newline at end of file diff --git a/docs/High-Level-APIs.md b/docs/High-Level-APIs.md new file mode 100755 index 0000000..5433e03 --- /dev/null +++ b/docs/High-Level-APIs.md @@ -0,0 +1,93 @@ +# High-Level APIs + +## AutoModel + +| **AutoModel Variant** | **API** | +|------------------------|---------| +| AutoModelForCausalLM | `liger_kernel.transformers.AutoLigerKernelForCausalLM` | + +This API extends the implementation of the `AutoModelForCausalLM` within the `transformers` library from Hugging Face. + +::: liger_kernel.transformers.AutoLigerKernelForCausalLM + options: + extra: + show_docstring: true + show_signature: true + show_source: true + +!!! Example "Try it Out" + You can experiment as shown in this example [here](https://github.com/linkedin/Liger-Kernel?tab=readme-ov-file#1-use-autoligerkernelforcausallm). + +--- + +## Patching + +You can also use the Patching APIs to use the kernels for a specific model architecture. + +| **Model** | **API** | **Supported Operations** | +|-------------|--------------------------------------------------------------|-------------------------------------------------------------------------| +| LLaMA 2 & 3 | `liger_kernel.transformers.apply_liger_kernel_to_llama` | RoPE, RMSNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| LLaMA 3.2-Vision | `liger_kernel.transformers.apply_liger_kernel_to_mllama` | RoPE, RMSNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Mistral | `liger_kernel.transformers.apply_liger_kernel_to_mistral` | RoPE, RMSNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Mixtral | `liger_kernel.transformers.apply_liger_kernel_to_mixtral` | RoPE, RMSNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Gemma1 | `liger_kernel.transformers.apply_liger_kernel_to_gemma` | RoPE, RMSNorm, GeGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Gemma2 | `liger_kernel.transformers.apply_liger_kernel_to_gemma2` | RoPE, RMSNorm, GeGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Qwen2, Qwen2.5, & QwQ | `liger_kernel.transformers.apply_liger_kernel_to_qwen2` | RoPE, RMSNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Qwen2-VL | `liger_kernel.transformers.apply_liger_kernel_to_qwen2_vl` | RMSNorm, LayerNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | +| Phi3 & Phi3.5 | `liger_kernel.transformers.apply_liger_kernel_to_phi3` | RoPE, RMSNorm, SwiGLU, CrossEntropyLoss, FusedLinearCrossEntropy | + +### Function Signatures + +::: liger_kernel.transformers.apply_liger_kernel_to_llama + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_mllama + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_mistral + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_mixtral + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_gemma + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_gemma2 + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_qwen2 + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_qwen2_vl + options: + extra: + show_docstring: true + show_signature: true + +::: liger_kernel.transformers.apply_liger_kernel_to_phi3 + options: + extra: + show_docstring: true + show_signature: true diff --git a/docs/Low-Level-APIs.md b/docs/Low-Level-APIs.md new file mode 100755 index 0000000..03cfcb0 --- /dev/null +++ b/docs/Low-Level-APIs.md @@ -0,0 +1,133 @@ +## Model Kernels + +| **Kernel** | **API** | +|---------------------------------|-------------------------------------------------------------| +| RMSNorm | `liger_kernel.transformers.LigerRMSNorm` | +| LayerNorm | `liger_kernel.transformers.LigerLayerNorm` | +| RoPE | `liger_kernel.transformers.liger_rotary_pos_emb` | +| SwiGLU | `liger_kernel.transformers.LigerSwiGLUMLP` | +| GeGLU | `liger_kernel.transformers.LigerGEGLUMLP` | +| CrossEntropy | `liger_kernel.transformers.LigerCrossEntropyLoss` | +| Fused Linear CrossEntropy | `liger_kernel.transformers.LigerFusedLinearCrossEntropyLoss`| +| Multi Token Attention | `liger_kernel.transformers.LigerMultiTokenAttention` | +| Softmax | `liger_kernel.transformers.LigerSoftmax` | +| Sparsemax | `liger_kernel.transformers.LigerSparsemax` | +| mHC (Hyper-Connections) | `liger_kernel.transformers.LigerMHC` | + + +### RMS Norm + +RMS Norm simplifies the LayerNorm operation by eliminating mean subtraction, which reduces computational complexity while retaining effectiveness. + +This kernel performs normalization by scaling input vectors to have a unit root mean square (RMS) value. This method allows for a ~7x speed improvement and a ~3x reduction in memory footprint compared to +implementations in PyTorch. + +!!! Example "Try it out" + You can experiment as shown in this example [here](https://colab.research.google.com/drive/1CQYhul7MVG5F0gmqTBbx1O1HgolPgF0M?usp=sharing). + +### RoPE + +RoPE (Rotary Position Embedding) enhances the positional encoding used in transformer models. + +The implementation allows for effective handling of positional information without incurring significant computational overhead. + +!!! Example "Try it out" + You can experiment as shown in this example [here](https://colab.research.google.com/drive/1llnAdo0hc9FpxYRRnjih0l066NCp7Ylu?usp=sharing). + +### SwiGLU + +### GeGLU + +### CrossEntropy + +This kernel is optimized for calculating the loss function used in classification tasks. + +The kernel achieves a ~3x execution speed increase and a ~5x reduction in memory usage for substantial vocabulary sizes compared to implementations in PyTorch. + +!!! Example "Try it out" + You can experiment as shown in this example [here](https://colab.research.google.com/drive/1WgaU_cmaxVzx8PcdKB5P9yHB6_WyGd4T?usp=sharing). + +### Fused Linear CrossEntropy + +This kernel combines linear transformations with cross-entropy loss calculations into a single operation. + +!!! Example "Try it out" + You can experiment as shown in this example [here](https://colab.research.google.com/drive/1Z2QtvaIiLm5MWOs7X6ZPS1MN3hcIJFbj?usp=sharing) + +### Multi Token Attention + +The Multi Token Attention kernel implementation provides and optimized fused implementation of multi-token attention over the implemented Pytorch model baseline. This is a new attention mechanism that can operate on multiple Q and K inputs introduced by Meta Research. + +Paper: https://arxiv.org/abs/2504.00927 + +### Softmax + +The Softmax kernel implementation provides an optimized implementation of the softmax operation, which is a fundamental component in neural networks for converting raw scores into probability distributions. + +The implementation shows notable speedups compared to the Softmax PyTorch implementation + + +### Sparsemax + +Sparsemax is a sparse alternative to softmax that produces sparse probability distributions. This kernel implements an efficient version of the sparsemax operation that can be used as a drop-in replacement for softmax in attention mechanisms or classification tasks. + +The implementation achieves significant speed improvements and memory savings compared to standard PyTorch implementations, particularly for large input tensors. + +### mHC (Manifold-Constrained Hyper-Connections) + +mHC implements fused Triton kernels for Manifold-Constrained Hyper-Connections ([arXiv:2512.24880](https://arxiv.org/abs/2512.24880)). It wraps an arbitrary layer `F: [..., C] -> [..., C]` with multiple residual streams, constraining the residual routing matrix `H_res` onto the Birkhoff polytope (doubly-stochastic matrices) via Sinkhorn-Knopp iterations to stabilize training. + +The `LigerMHC` module takes input of shape `[..., HC, C]` where `HC` is the number of residual streams, and performs: + +1. **Coefficients** -- Compute data-dependent routing coefficients (`h_pre`, `h_post`, `h_res`) via fused matmul + RMS normalization + Sinkhorn-Knopp iterations. +2. **Pre-aggregate** -- `x_in = sum_i h_pre[i] * x[i]` (shape: `[..., C]`) +3. **Layer** -- `f_out = layer(x_in)` (shape: `[..., C]`) +4. **Post + residual** -- `x_out[o] = sum_i h_res[o,i] * x[i] + h_post[o] * f_out` (shape: `[..., HC, C]`) + +Usage: + +```python +import torch +import torch.nn as nn +from liger_kernel.transformers import LigerMHC + +# Wrap a linear layer with 4 residual streams of dimension 256 +layer = nn.Linear(256, 256, bias=False, device="cuda", dtype=torch.bfloat16) +mhc = LigerMHC(layer, hc=4, c=256, phi_dtype=torch.bfloat16).cuda() + +# Input: [batch, seq_len, num_streams, channels] in BF16/FP16 +x = torch.randn(2, 128, 4, 256, device="cuda", dtype=torch.bfloat16) +out = mhc(x) # shape: [2, 128, 4, 256] +``` + +Functional APIs are also available: + +- `liger_kernel.transformers.functional.liger_mhc_coeffs` -- Compute routing coefficients +- `liger_kernel.transformers.functional.liger_mhc_pre` -- Pre-aggregation +- `liger_kernel.transformers.functional.liger_mhc_post_res` -- Post-aggregation + residual +- `liger_kernel.transformers.functional.liger_mhc_apply` -- Combined pre + post_res +- `liger_kernel.transformers.functional.liger_mhc_forward` -- Full forward pass (coeffs + pre + layer + post_res) + +## Alignment Kernels + +| **Kernel** | **API** | +|---------------------------------|-------------------------------------------------------------| +| Fused Linear CPO Loss | `liger_kernel.chunked_loss.LigerFusedLinearCPOLoss` | +| Fused Linear DPO Loss | `liger_kernel.chunked_loss.LigerFusedLinearDPOLoss` | +| Fused Linear ORPO Loss | `liger_kernel.chunked_loss.LigerFusedLinearORPOLoss` | +| Fused Linear SimPO Loss | `liger_kernel.chunked_loss.LigerFusedLinearSimPOLoss` | + +## Distillation Kernels + +| **Kernel** | **API** | +|---------------------------------|-------------------------------------------------------------| +| KLDivergence | `liger_kernel.transformers.LigerKLDIVLoss` | +| JSD | `liger_kernel.transformers.LigerJSD` | +| Fused Linear JSD | `liger_kernel.transformers.LigerFusedLinearJSD` | + +## Experimental Kernels + +| **Kernel** | **API** | +|---------------------------------|-------------------------------------------------------------| +| Embedding | `liger_kernel.transformers.experimental.LigerEmbedding` | +| Matmul int2xint8 | `liger_kernel.transformers.experimental.matmul` | \ No newline at end of file diff --git a/docs/acknowledgement.md b/docs/acknowledgement.md new file mode 100755 index 0000000..9dfdb4f --- /dev/null +++ b/docs/acknowledgement.md @@ -0,0 +1,23 @@ + +### Design + +- [@claire_yishan](https://twitter.com/claire_yishan) for the LOGO design +- [Wave Snippets](https://www.wavesnippets.com/) for generating the animated code snippets + +### Code + +We referenced or used the following projects: + + +| # | Project | Description | Location | License | +|---|----------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| 1 | [Unsloth](https://github.com/unslothai/unsloth/blob/fd753fed99ed5f10ef8a9b7139588d9de9ddecfb/unsloth/kernels/utils.py#L43) | `calculate_settings` to determine block size and warp; We reuse it for Norm and MLP | [Liger Kernel Utils](https://github.com/linkedin/Liger-Kernel/blob/e249eee723978bf8610ff1ea2297d048a2417e20/src/liger_kernel/ops/utils.py#L23) | [Apache](https://github.com/unslothai/unsloth/blob/fd753fed99ed5f10ef8a9b7139588d9de9ddecfb/LICENSE) | +| 2 | [Unsloth](https://github.com/unslothai/unsloth/blob/976d11a10d54383aeb7a692c69e01151a20bfd72/unsloth/kernels/rms_layernorm.py#L48) | We modified and added dW calculation on top of Unsloth implementation | [Liger Kernel RMS Norm](https://github.com/linkedin/Liger-Kernel/blob/e249eee723978bf8610ff1ea2297d048a2417e20/src/liger_kernel/ops/rms_norm.py#L50) | [Apache](https://github.com/unslothai/unsloth/blob/fd753fed99ed5f10ef8a9b7139588d9de9ddecfb/LICENSE) | +| 3 | [Triton tutorial](https://triton-lang.org/main/index.html) | We modified on top of triton tutorials | [Liger Kernel RMS Norm](https://github.com/linkedin/Liger-Kernel/blob/e249eee723978bf8610ff1ea2297d048a2417e20/src/liger_kernel/ops/rms_norm.py#L50) | [MIT](https://github.com/triton-lang/triton/blob/main/LICENSE) | +| 4 | [tiny shakespeare dataset](https://huggingface.co/datasets/karpathy/tiny_shakespeare) | We use tiny shakespeare dataset to conduct convergence test on mini model | [Liger Kernel Convergence](https://github.com/linkedin/Liger-Kernel/tree/main/test/convergence) | N/A | +| 5 | [Efficient Cross Entropy](https://github.com/mgmalek/efficient_cross_entropy) | We use the idea of gradient-in-forward and chunking | [Liger Kernel Linear Cross Entropy](https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/ops/fused_linear_cross_entropy.py) | [MIT](https://github.com/mgmalek/efficient_cross_entropy/blob/main/LICENSE) | +| 6 | [Flash attn](https://github.com/Dao-AILab/flash-attention) | We take many optimization ideas from the work, such as tiling and recomputation | | [BSD](https://github.com/Dao-AILab/flash-attention/blob/main/LICENSE) | +| 7 | [AutoAWQ](https://github.com/casper-hansen/AutoAWQ) | We reference the design of automodel | [Liger Kernel Auto Model](https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/transformers/auto_model.py) | [MIT](https://github.com/casper-hansen/AutoAWQ/blob/main/LICENSE) | +| 8 | [llm.c](https://github.com/karpathy/llm.c) | We reference the design of end-to-end testing | [Liger Kernel Convergence Tests](https://github.com/linkedin/Liger-Kernel/tree/main/test/convergence) | [MIT](https://github.com/karpathy/llm.c/blob/master/LICENSE) | + +Many thanks to the contributors to these projects for their invaluable work that helped make Liger possible. diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100755 index 0000000..8388f6c --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,114 @@ + + +Thank you for your interest in contributing to Liger-Kernel! This guide will help you set up your development environment, add a new kernel, run tests, and submit a pull request (PR). + +### Maintainers +@ByronHsu(admin) @qingquansong @yundai424 @kvignesh1420 @lancerts @JasonZhu1313 @shimizust @vaibhavjindal @tcc0403 @momochen + +## Interested in the ticket? + +Leave `#take` in the comment and tag the maintainer. + +## Setting Up Your Development Environment + +1. **Clone the Repository** +```sh +git clone https://github.com/linkedin/Liger-Kernel.git +cd Liger-Kernel +``` +2. **Install Dependencies and Editable Package** +``` +pip install . -e[dev] +``` +If encounter error `no matches found: .[dev]`, please use +``` +pip install -e .'[dev]' +``` +3. **Install pre-commit hooks using [`prek`](https://prek.j178.dev/), a `pre-commit` alternative built in rust** +``` +prek install +``` +Run pre-commit check without committing (`-a` is equivalent to `--all-files`) +``` +prek run -a +``` + +## Structure + +### Source Code +- `ops/`: Core Triton operations. +- `transformers/`: PyTorch `nn.Module` implementations built on Triton operations, compliant with the `transformers` API. + +### Tests + +- `transformers/`: Correctness tests for the Triton-based layers. +- `convergence/`: Patches Hugging Face models with all kernels, runs multiple iterations, and compares weights, logits, and loss layer-by-layer. + +### Benchmark + +- `benchmark/`: Execution time and memory benchmarks compared to Hugging Face layers. + +## Adding support for a new model +To get familiar with the folder structure, please refer [here](https://github.com/linkedin/Liger-Kernel?tab=readme-ov-file#structure.). + +1. **Figure out the kernels that can be monkey-patched** + - Check the `src/liger_kernel/ops` directory to find the kernels that can be monkey-patched. + - Kernels like Fused Linear Cross Entropy require a custom lce_forward function to allow monkey-patching. For adding kernels requiring a similar approach, ensure that you create the corresponding forward function in the `src/liger_kernel/transformers/model` directory. + +2. **Monkey-patch the HuggingFace model** + - Add the monkey-patching code in the `src/liger_kernel/transformers/monkey_patch.py` file. + - Ensure that the monkey-patching function is added to the `__init__.py` file in the `src/liger_kernel/transformers/` directory. + +3. **Add Unit Tests** + - Create unit tests and convergence tests for the monkey-patched model in the tests directory. Ensure that your tests cover all functionalities of the monkey-patched model. + +## Adding a New Kernel +To get familiar with the folder structure, please refer [here](https://github.com/linkedin/Liger-Kernel?tab=readme-ov-file#structure.). + +1. **Create Your Kernel** +Add your kernel implementation in `src/liger_kernel/`. + +2. **Add Unit Tests** +Create unit tests and convergence tests for your kernel in the tests directory. Ensure that your tests cover all kernel functionalities. + +3. **Add Benchmark Script** +Add a benchmarking script under `benchmark/scripts` using the naming convention `benchmark_{kernel_name}.py` showing the performance difference between the Liger kernel and HuggingFace. + +## Run tests + +### Use Makefile to run full tests +1. Run `make test` to ensure correctness. +2. Run `make checkstyle` to ensure code style. +3. Run `make test-convergence` to ensure convergence. + +### Run pytest on single file +`python -m pytest test_sample.py::test_function_name` + +## Run kernel benchmarks +The `/benchmark` directory contains benchmarking scripts for the individual kernels, demonstrating differences in speed and memory usage between using Liger and HuggingFace module implementations. + +1. Run `make run-benchmarks` to run all benchmarking scripts and append data to `benchmark/data/all_benchmark_data.csv`. + - Existing entries that are the same (based on `kernel_name`, `kernel_provider`, `kernel_operation_mode`, `metric_name`, `x_name`, `x_value`, `extra_benchmark_config_str`, and `gpu_name`) will not be overwritten. +2. Run `make run-benchmarks OVERWRITE=1` to overwrite any existing entries that have the same configuration. +3. Run `python benchmark/scripts/benchmark_{kernel_name}.py` to run an individual benchmark. +4. You can use the `benchmark/benchmarks_visualizer.py` script to generate visualizations from the CSV, these are then saved to the `benchmark/visualizations` directory (note: this directory is not tracked by git). + +## Submit PR +Fork the repo, copy and paste the successful test logs in the PR and submit the PR followed by the PR template (**[example PR](https://github.com/linkedin/Liger-Kernel/pull/21)**). + +> As a contributor, you represent that the code you submit is your original work or that of your employer (in which case you represent you have the right to bind your employer). By submitting code, you (and, if applicable, your employer) are licensing the submitted code to LinkedIn and the open source community subject to the BSD 2-Clause license. + +#### Release (Maintainer only) + +1. Bump the version in pyproject.toml to the desired version (for example, `0.2.0`) +2. Submit a PR and merge +3. Create a new release based on the current HEAD, tag name using `v` for example `v0.2.0`. Alternatively, If you want to create release based on a different commit hash, `git tag v0.2.0 && git push origin v0.2.0`, and create release based on this tag +4. Adding release note: Minimum requirement is to click the `Generate Release Notes` button that will automatically generates 1) changes included, 2) new contributors. It's good to add sections on top to highlight the important changes. +5. New pip uploading will be triggered upon a new release. NOTE: Both pre-release and official release will trigger the workflow to build wheel and publish to pypi, so please be sure that step 1-3 are followed correctly! + +### Notes on version +Here we follow the [sematic versioning](https://semver.org/). Denote the version as `major.minor.patch`, we increment: + +- Major version when there is backward incompatible change. +- Minor version when there is new backward-compatible functionality. +- Patch version for bug fixes. diff --git a/docs/images/banner.GIF b/docs/images/banner.GIF new file mode 100755 index 0000000000000000000000000000000000000000..a6a3f63030044efd0dd184a15f4198fab7b085d4 GIT binary patch literal 716878 zcmeFZcT`hv)FztVNJ4<1bVF0Rp@$*_2pu&v0Ywc(ML-N72!fbGFA}6T4Mhk=2p|fG z5UPL;1Pj&x7DSXNsOV44^}93Qd^2;`H|wr7bLY;iHD{f_&U(*T`+eTM_q)&8=Xso6 zok-?E_aJV-?|}dNfGPrDsz8`7Ktc(C1_RJ|Bw81VwgO<#0E{LWV+_FR0A#i!W%L1f zFaR$Bz~f>3-*#e}h4^=gRsj0)%EFkJIs5%~|u834OLupu{GA4*=b_=v{KWO3+ zWZ@BFyw|~qx>L^cpov$ArB|Spcc`{+$S&XGcKgBu_MNr%quBW!wemNx2{>vK5aF;t z()d7x!vR01gVC-BV;l}SA2`I;IUM17ILe6@9{;6mWvME9#6_l->T5DaTLd`q5MNMSJawW(UTyBVvmKPn|pv zclc0TR#048NPJdsVtQO+xyzaK;H1N$NhgmaFj>ksHA{AHh0zf>juv zdh$qGQFz*+n6xtg%;Na$>QgzaGkG<~&R6E;x5pRI<1YlKT&PbfVzG*wPLi+xZSqnN=W^fe`hH$Z|Gg_i4=RW67LMHO9C>(U zwC&1hA9w6Q>ln9pe7^hEeAm>?zT3~acOG`#X>Gpy^y=NKSMSx<+@HUBe_`nULf-@a zjR!YJ=H84wTAp~cJU+iNIsfkV^N-W7KHn9t-4lM95w1OWz47GTx5vwW&mUs3_^1fl!U1A{-oFILzk~?@AqF6|xMV(eh>eiZ37p{9jAY={ z-KtzVYR7WqjKiN#bkyC-Cpcs)xpvl17HWAn2TpcgzFkTTnW}Q7+Oc6Ku`ACfyBhCb zBC}D-ZdZ9TbuNWE2dAz!%{Ef&+^XHWn;*9LcZI*0>TY@5P8-QqcE8p--xW67eDL)m~>LypSszN_zNdEPCD?)G(mc+3yEeR21VYagHW#C~{r_r~?LR|2-gwmtnl zpO+>Ibr0X`?_Ga4UFUvj&&|HCAD(nYyt;Su#`iVhNRD{h-huuf>&vq(hwl&E{O7x9 z>Gq|)g9E?+*i*#1UnpT(LIz|@n2?Fs&J(iGX3Ii0&Pi(_6~8ZRAx-WmZy{YN zet98-kgm0usZkQPn5EUgTg=wGw!D}_9M@XPHF*@alt+5YTRLz3ZFwo54AXwip=>$+ zy1-?7)9VW!X767YQk}Hl6#48s{-)UfXw#dL!1(uXN@?lZZ_7eUj=wDrYiN2~5qa(X z+e-Sl_PeT6kB+~qPI%k&?qc${_wOz-U^>g()Ggu5HJRI+muqv)R+j5HPCD=Fi}r=T zzg%{-`F%rG{L1@AZo1A2uf8OFrHR+jywcoyZDpl}KQ7k!(0cVz_=hV!Z<{}~^?zIW z&@O=KuJT8>M67m9Y;RfZykquZwQJf*_v6*MeGwnKpB!!ZcAIhKUYA6C z>RoPV`P8?1?Zc-VqH*1|{*6ZwYd1IFwyX{O`u1UM5CGQ`2_Rb|MMH=kt)gMH`Ko9H z=dAa66z>=Lc}(tD>*sN$)2p9v5i<0?OlXuwewoy2Z2dB&cYXECZQ?Dx^*bhyBiHYe z-nFjZv;MxiexD52-W}*Hd#o>YD!Yy@4~p-LmMbaZt3& z+z|4>`W#T{Izs)9z8~Hv{^2jPZxf_+4`ms!d~OWLF34T{I$hODn`5WG$&cM_JiqEm zeMSFn^$n}J{2F4phhifI9+q4GX0IH1@ASLe(-29kP-6lBs+{`K!lBxG?ueMtDcHNr{u#=~9hSmh)dr7d4TYIQ*S=`}aCq4-_H_+G zUJGc6FvOg?Q!4>obwnBw!1N-J^fUikmdS=gkfTEEV=~-QU??Hqb3*av1VHzys=X8U z69ha*Fn`pM_s60h)Xa0X{mINvoEwq*tUj?*EUaO;3mmrn-b@1x?*j%Xd2SQQrGa1% zD9JdcoONi%BFrABga}djJ?(5J60kP&YE?c9G87)WKnAMq6gSC>rVTg>>K98H#erCD z)JST22^FsQq}xmxrbcdrD4mR;m~TYqCkkrg3S?l`mNZK8#>iG)2oN0jnG#>Nc<#mu z8Tmy7r6j5Spj8r5(hqz^%@r-i`@ooTz}^`=}0O5Z+Zj@cgI>=l$kQOB&C+A z3wELc{cv>=C;@X=!13Ufv;*$; zVn~MhhquAJSmuSt&W?#=E%DD*{pv2E0VPe}X=4TyQ346m7mTQ33TV9*B zJhuNZW7ZG2N1xF52qF zek7zO|7cMMVXv{%E`!gZYsbIud;Yb}=W)xFm;b|lO|7+UE(Gu?Iv?DvJ9faGpZ*J# zF9z-Lzoq->cAlxIcx-?T`PyJ1l^nJ+h33w82D7b)5Ki#UCn0j$XtTH#HsWUsKxs4p zM_TPrnc>DLfA^D=*?T26U()x6U zuB8A3y|XC|4OpDbIHjN)!HzYZ+Es z+ERGB(IL;@9y2)KHwP*ItWN#CVtbVRaI(j>Rgt(9>ar-EA4ZwDPjrL=K-<9Lca_g9 z>97DjX%!ar+Lg}asUZ6scufGP?8?ETK|k9dtsG6*xPCu&ZmV>tF0*%S)oQM(@`Y6N zY1}(`vqba7QHj%Ft667uO2)@m1zQr7OXX}G=9ZG9eG`67#e)(EG10#>qel<+^}Uy_ zf7pJ$8un2ZZ z*CfCSCie;QJ_Kk(%zC$p1aGtAA-9PPQQq#gpXM$DUTX8BKaQx6qG&+z76afD^lFVt zy0f*NC3E_?jreod3R&e5)h()@^(TVF1VtNwV||j8QyGW8y@LoejvXija)j=JQH*q| zB_8Mgs1PSl+;x)=i{QXRaZ<`uOhz0mnvab31t#$W&;(l&Rq~AhN}-`1b3BEQJxK!d zFrgRE0rHcE_NIe${S!{dfi^i3NJHpXIwp`0P7<=9sweX~xW{zNXJagx0$is{KvWUL zbf^g*zC^|9k*)E9(5lNRPyIYUvEl2uP|-V`pgMpR5&MCLRj9XHCSq#qF_))-vaju< zoe)1Wu4GOBW-sCDVR*GVF!G;V0KK8Vdi(KwL9F z(EUhqTmaI4DCx%qT<1y}%K1l=kMlwhV-1p<0Lkjr8I|BNNu=I9_ z>vh?S5`p}PyOY>Q-_M#~GPB{STT`g{h^sN1Pk`Puhi6*4M`Zlq%X#f^>x4|~P2z>l z5Wm*b36jhZQ$r7YFR6A5>z4O!s(7FERTrDJFx>khoWZwU>HAoA8n%x^jz};Q!oD5d zrmaURFRH!j&K#3dH7wj+uD=&q{FsVrKV5t%QOdEj_^?F@1{gH-$S1VCL{il@e7+>= zM+sfNG{&OzR6uEbVrfEo>6zZrH{hovXrZr@zt4e5s4MgE>w7hB>CSe zLD*HcHEHiSDiv8?DK}KfPjt98FE!MJ8?dMxOsuROu8{4h=ueayPpp!dsZuqp8vRlA zAV4?Wsd|4<^@WMxr%hX5A*JKR#->e8RSKV@;zD=tl&V@S$1RlG?p?pA7gW6waH;Eu z)R*_w-x8}gkzp?bDt{rl8`?$Jj$dSqS4(}~>sWq?dad$}eD(MDx@ZL%8C2Eye08*H z<-H%f{`6Lv`xHe0t0T%Ub$ZwA=(wcyv&yKTb{c7xcMWG=bD~e43p!JSnYzfg;3C6o zMTuOrT{Y>m>C^IR6H6VnK3uT2w6&!)ygVr5bCIpKRNev&V^_q2pTumx*p0xCvvDq2 z_2{2wn*L_)eKPM0FSC6cM&C$GBMf!Qb-&8U;+Kb2+;f$BsOg9LOPzu8yna+|}&>lL_B0PB4( zj!#C9)6iAL(l3oK28XrWdD@D2$enR*FxScIn`#+AwOV_gRIfoza`SE`;`khi_jox8 zD$tgO3Y(TIs6h5}UcrX(U_3|+|7{KN2qh?tAyytVK}G`D(}cVK!$^A;_?l` zo3>O`Vk%KN`>TtlDlk+;2MttlhTEG6?FXO*3<)Vf*fln=Um#)32O^$9NK*c?G*lS_ zz!SUGpC7K9Ct_zSaK}T?zv%%wb`YN@$tf$+%2e!6F6OS_>W+$w(EXK8?ovBwsA^G- z4HBMDt3!8ytLex-XW%g+icXWV^)IoJPg9Z|JjR(na)2tGj~ zomQMNRpWKWTA7RA7yEiAy!0GAk&^9<5CSkGj*w9xHj z=d_c5pA>iqaF^z&5#$9T?v>>OBnZH923ndAfDG|FIM`Sru!fG!;sRwDI1C?nn~YH; z0uQ=2iDJR^(~UbG0bnBZO8{{2Sq*{-j3r~!7(nkVa1+1%{QkN;arDhJdSh72#jTdW z77^AM&@p$wd7nU{hm83oyaAm7NHH*L43G*D{aOS`&X$n?z?X$^1Oq@pwVn0iD)YOj z|I~RfB=hT~^Qe;h#_L=aE=5cQSG+;7zT#Xfq?Vd4Dnui#@v*kqxIr2^(7j4MQxX9+ zo1b>xQb2J&P^tDqaLF+&=a#eeoe}uwq2*puG)|HVk=*2BOoTOSPJA`J?n;JahTO8Jb$oF=<1^|<@!-o~ID!9Ks z&wmy^ILAj1d`L?ZVI-;jNU6FCbSKLYcw0EB$e(CXYD(;zX#5p!hyq>_Vr}S+$-d3- zA)pFPVnqo2XNs>RLcbRRRRHMsbjX#!J_(M*9T6xO+R5&;Snx9=;Syk;RuoQllRC(I z&f-WtAlljyHe@3)1Uc{y*F0jO5(w-ix_T!i_)s_pNjO$4*ogu5u6G3q0pMs$SsEnc zv1vr3tKRhZQThEJ>PPNu^!-MW^-f`@nok7vTCuIPVd6wQ(P~hk({fCdD970eFsINrH`<5u@(1a?P zgO;X{sm%M9N@jWjNTNO|<22ZWYIS7EwBHOTgiF2jw+h2>gJ$oB?Pu?(e!MXn>d*Ha zemVm&gm}~Mj+IV3UAyak)slM!U_b|(N?FR1A)Oh=)CD%9g_uQL*p+`kw>T2eUnXc> z)F%!kr4<_|ykR7+0Lt-!_xb2j4?xWnP>B!RJ>7ZwDZsnvZ`BRf76EWn-gP!m3McWB zJGk-t#y2Z;$tqBr3(^q{+9&|igxGN53k(<3LYtJNPX?RfL^pvJ9FXGl)J=9=Rx|u1 zUznT?MDl@qiBk$humTMvC$y5|fV3GP1uhUtZpLt4+K2#J^xI~*zghaFA`YU%0aPf- z$#H>#+-N#M=tX$3+g9&^1#4CnyBkw%pvkl>zG%W!)GL))S(7VUBGK%ajwNk%yK=LW$6=;8tkP z3juud4nW2O;!n-`D(Kn{kBxkZkZF~yC*!sdK$C*EzquHVIO{sDq#pz7X5}*T5K(&I zz#tbaH3aaSmaL*nY~q656EV7P8GNdw6Ah>*LRxMt+fpv_IT7h`-MrYj0rC)Dhpd0}kEYv$qMbh5x=6CfRueClX00 z(*a+GPPpVLUKvl260uzNjtkoxuoeN|Hv3tK?(TyPKuc!&7D^AZDo`8L35dL1H2Kw+_(jTHdr@v6|I~Sb@o#(F=4oJRWK$RN+exa^M??LyD07&${-&AZe0s8_M z2I2$C2pAt)*IxDJav|1C1c(*>O>8d^slqi5Oq*!$DJhW2%(R}WbAw5ua7VKl^7U-5U4}z4-sKz+p!Bm z%o70+CbE(vpaf#<6E0Gbzqs)TU`+m_0>CMf$~NCF&)?X8h<|hBvZ@^n+th#4Jwx9o z8-r1xXBM{w^Y*Qr#}n*=VFoeUrD<8y4Er<`-fYh6U5#cM3}yzXdpqQye^SM$FH%5e z=^gdiA_#RGdhgBiyu9b~{)a2RtpVLpZ=OE<;KvUdJFj`*(ds<*`ig0+@y=)MBvFjT z$7#$(f?m{xhq1Fv7VBnRgK)-7@(>(~hrEy2mp;Zz9Ch0~J4LzFS{88J-~Rf&^>=Y& zTGWf#a?W+bd1zJX*Ct&9hiAsi?ZqZj?I827wZ(>@h)d1VCsImTZ-hCYmE;dek$na6Zx zFo(>DQ`$Q^(47{NNb`xw9jH7@ttAyNVOB$dvrFPiZemn93t_k7y1fNuNh6n>p=&u@ za{i*YvoiYP3Si6G1}=lCWjXg&+D<`R-58ZJlOdzlFyXk(&uOXd66u~dSH)}U$D5tR zmC138x`+W+*>Y5=wmJrDxL6QHbI%1)ejbPAYI`v6w#x*ngSG{5S!+hkc+rfk9)I)l zCACZ84NATny>Bk;;nUX?#AkzDa?{Xmo9+QR-|a!T!>iLWf%qY|DJxMfa;#^||uS zLi=sL!ls5oPp8CYspQP)kW6=x;kdSqsP}Dd_>-oF+^DG;9mSst^9uzjToU0>=zeL?`h&ixW;N(MLlCyo2#}0OsZ3VH(SR!ft^{d|(mG zMh3=&v*h_eC$Jb{Bp|Sk^m`sXzeC#Mm^|Lhg^+iW@nbR5_vLX%FcuM}CUs;^UZcR` zt&j@@nfQ+@L`YlM%bZM~2Y^dy;bi>jfdm?B@Tk_PLhpw%2T2)(x$2e}ip65i{s&0S zx<;Kca^b0Aau5h>mH@A`wD)q#+kfwK5E$W==v=7YpLtUnqgF=1cjqJ2=Dw$aEk;Ms$_g8nmBIF^8&61)`3%0F@#JeL<-TU^kRYxQ($B)zRviYv@?kN$V{}4 zLmCP(qs}7^mkDaO1w7zp`il_BF2mlI)6S+KV#VR1C0Ej9e#*CK320C7w!8@jdCI9hZ1pzB1tiUMClMLgr%hSj)W9$xLd3&sGvww+96Yp1=3=+e$WxO zLv=Nbicph^GaK3!`J95oQ8SIwgj$t^%!jSvak|aV7l~h~P`y4T_-tsQ&a2r8eNi+~ zta6EVS!q5^18@UfI{ARnkp?Y8+< z0Zz?WK_^oiEGw|zHPvC<_)92v$)5o!0a4JQ0?6@c2FeQu!;tg;mKF@HqDNK>8VrR` zJoa8H&dQWlFWEz~|3(#+60N5dWCW2PC zMUq#MmPgUSup^(@wdYY^OYD2*+46 zm3l}OdyAy7cntvP$?{~7xcR(B7$a;UGZju10$;$`C)x@Q?T%u}NJAEoxM=~pf^MxC z{#tVIQt@lFuRgxOOx-IdTIyx7_o;_=s_drn$&@K0!x-sr-BL7 z!d^y0dp;RC-dQ93u>Ge^eIOLXAvEji;-+Z|kspNzwaAmFO%(DzT z;LeA3`9C|Bo@s=TNkL4Tm6;>FxzAE8HG53E)?%}en%Am9-_#PQeSe+d{pu9T=7Qlx z*>MW6@Clrt<}_oZMKko~JlttBmlvqfprZftcG!kg-mgs;_`(N<)VBpUlpJ%<9&`gg zYMvQ5^&b7Z$EjQ8D9R!AB>scO?t=R`pYF0n?ZJMy^*E(s>#?SEO`4sODCmku+=yrL zbK`?kKuK$I?0No20V<|0JAQnjy5~Z+o^Ylv0)V>ovkGOaD7-I6k9DfE0NSW%$?6H0 zwqW8Ul|;-FN)J%ki-u!+yMhS>%gd$W>V#~_LC>MjS`4{Ou3d@r(zdBnH2m+(CLbCF zMVcMkwR1SbCCD9!1jSXvd6tqbj--046nm04_JS>HBq`Hrpnb~s?@d!Nj;Bn+_RdT| zk_UjVEcC%LRHk`CWkHV@xVP#dS>d@9ve{0^mn1VWEGeucU68H-7`L#d0uVm?NK#y< zy`7D~>o%(DhUH|Pmu`$wA1*NPRv$Ic=#+C$o#caR0L1>YHTo1lt%c-#YHghPtPOPI z9WNE$&pmIGK#=-W#)d!m3D{lbfPp5kto8>bV#5SmYToD>JP%^829zeKgzevGxX!%2 zNa5Qw{{GJG1RUK&4{W{>pItjEP7=axQ#NMd=Oi6~)I28(jYrOvPKGV0^L2_@HDDXz z5O&p}%tdvM=oct9d?Y)zd%UxuG1&r*d|!6y(c4n;y5j+aXykoff%Kr}C8Q zh)K^?E~Bb#SF6W{!OkMiAAgzy49(SwMdcB&x(=yliFktuhRO};1TFZPY4}-so$Qm0 zFiy6iAq;y5=AX&>MeLAO#02x;L}O55m$5aQOJSjG!6uIrr2dhY@o9$JEHfNKm@y0s zC|%k;BAua#`T3+=g?+^cY?_q~kLSVT8Svy~c%)UgPZDZV08i$@&o0A_XVYh>h2GKG zr4bmymmOk*C*ATt>JsH-Z2IyB%UGdh8A$>-mU7KX5q`XuLH^Nz6@cTHGZJ0Wow#Ta zfnkI*CcY+nBUDiw*%NHkfrV>4(KR#I2%YudbSG6HlA(iox38LO4{#y7OH>PXEDDRa@-aeIjPZ@XX> zo?oZVQRg34>vanO?8#!u8j;bdaAIqQ(ba9`g##kwHWL!emJC307%!GAVv|s4BGXp{ zQ`BPIaJ77V2Ji~WjzvH*S^&&6Bs?MGGzlJuxKa|vs1{i!*LQp-+_<=0jiI-Q)j4gJ zUzn$<{SEH{BMX-8g>5~R3^zJ_C#MbO0&pf}SkM%P&VWu*t+50pv`hb3Eaa9|#>8yu zXpkb>1#r;^j6i_zcilj8vy@zPY19n5meol?dTQv++bae)a!_A8cg@FEKuPH#0EGj1 z8$Q~-L&?3fYUgjA0hP}?QJXSvM&NH_ZI+WzvGt5Y_M=Wcx1BT3qi^K4pDK{8WoTYf1Uzf(G`$~rTx*+(ikz^9NUij3#E5p+wzb6I!tULs0?9{;=FICkBOhA*;v3pB0WYa%A^+P!cLoCu6f?!qU;3vN0jq z7@4tiUgqAb@WY$#c6IJyL5_Yt0RJM#{j0Yd{mTQDW*T{=s%6=Gnk(={4NEv=q$Ix8F(qlp5i<=s0z{=FxM+pS#uy5 zph(9qFgXgZaxy-8trs7~aO_>dMjEQI-;!5`d|4Z;oaaHi-{wdQF?|%REbg0zx#eV|wXlS2)#!xngieoA!JT~10zl$*aba2t zKu1+ka!yI6{BNa_kZ;Ii@&h{`3beJ&z8lfI@LgXEO0FNTDRMhV` z`e#F*SfJ#9-pd(*yd8d2sB|uPMXBo-Luz|wR#{K+=#!gklna^B*fOuHCVAP>fbK*D zf0Qrl(It~C2LefT>kl_7DNCxV=i%n<8%aQeWq5#aNGg~SF#9yFt6Sq>t?L8%oetn~ zbx<$?M23{O+NXvjp(M^`AFXUS{EQt5tR)YD5yM*~vQPmv-dkNgbBlNICK#%uCa)sT z7VGN@m(kv>8ITLvVqS%LMhrWtjzJj9F!M>fG}Xfc^hnrwHBxIecN?dD2*@a;WU-q< z>;0NsI#uZ!7cFJOmAtoV&$>y}bwhKdn-1}`rWsw$2i@+Mz!Se=i1ekq>eKX%X52==;mxp>bt;SW--=U0E*cRuOq(W z(b;~cu0^?kQHx0gfw`S#eQ)Jqg$LY|%+w&kgESvawpe3{K7#rFdl6}H#v=q5iDv-* ziObLsA(@n$C-bt(dC1b%?DgYMLqzfr&diS!e!2s}diNGL=i61Xh_uyhP=90ey2O@g zOe5~4_~m85f^Vs;0=(M;ZIX~$%qA4aq-}d3-`bfi`HxK-5q909EBodX9uFQI#>0}q zn=My&FG5nJ14Qprje9iH90uLobBcc3T77^d_duqo+96@sWHs0WLiX+pf%Et#axd`j zpAliv`G5()p5?K@2|%(-NcimnY>(O)mt=boK;l9@Hk;);B%Jrx%*(#nJCyOA%BX3E zf0;Y-LZrTDy$&q>CR;y4&(JJtv{@~(KD-L_c?qoXqH}i&%CsP@QX%90DU`u-s-A04 zeKp))4!Bi5(D#XtK|SC$M?-cpJs!YAGo;iKp1Z}e-PzezRUjx;&0MbV>vZ>^i>!D` zi%GPjDhqmI(01$9<32-XjhyX=X0GMWz=J13wiYy^F1)xLYzX560=Q5e7cEMy--{#H z_osK+D!^^OHd02|XL?K{J(pAJfNFQ~t8u6m!QPx^`0cSLV}fh0;i&^$iGUg|7p0?| z3)CMQo1p=9t-(jzLPNu+vC~D~Mu%@h?e$VoP2X~k{eXHTatw1c)p;-v+AZ;oY=uRD zX+j1?FxLRw=5QGnCj#LS6Z;Vvg>!(@T>abL;0B15!V-Wc=o`7{nn!EB=)#}hpCF!)qeG9;Rio=lRpoY=9jDRHt~ z<9#exiUy=`>fH886kh>Y%`(qF+)tcX3m5R`^Y<^nJg#>td;;d_Ec?b8e>8O z+Gu4!E}e9E0(O{w4Zp?SC7L)^o|BrL1>1xsFGJKbLHnoS+D4(Mli}s4%SUk#yRh_> z%+U)g@Q^1=Z9+O#Exo`?=CA;$M@TpSE{CG)1k+epE|cskMYyv^)g?Yi?%)>z*iHc6 z#)R4l*)sZFk=&ASc_{iu%waYv`Alj9fE~hv>2Qyomc8bcTeiO;+go}Zdc!p0N$10a-<6)*2IUJWWSWfDwTmk-M$+STqS9N?RXTZ77$qr)K`}F zB7n@7)2W>psDSwbl1ejySPQ@fIuQ=RhF%;e6oCP<+p>k9aJC21TLrfxu#Cy{_#W-L zFj&ZDxfCzBNehVfPrrz;$|GGa^67g#$+Qu{!ig8*nTn=-ri~z-Mx0zy1z`Q5r+KM6 zUD)`qX`Ya^gk{od>}uutrZ>kLD;pM}ld5*lF&gxh%!O&rDxJhJDY1 zRv36!8M{R>RUCDKCArUTQ@RtC2|iYq`eonACEJwOerswDxraB+U}5PerWjhwhK8%@ z#+x<20PJGJE+G?Gy&!LJ`p4nc5Q^ri<~}-zyK-AF6_(Fg*VGgABHPS|(l#LcBn|G!U79kF1}r zd8H54bTbR1I$h$z>OSqR;i3-XQJ7l9jYi3B{qD*non?ByKbVcPLLXUZxoDGqx2@>; z1VGAozr+?F9-!r=e3qsxr}>uaPcW@~z7++nY;;7Ew+iA@Jl{yTz7_y|1#c$2ML(@Q>3< zYxKNbaN)Tfur-C`;uK1_#%@j|u%q;rB|>i&jc*ita{;%$ByG|Tp1I8mt%~~n^F@ex z^T-ssbI+^wIQz20sDx|m$X7cRCuKjrE3Hh(QC`g>tsJ!UuuBEp@^NtVJILJbZmUPz z*dI3d+D{f+%3qqbzTQfdQu(@CLstJl?^yKaE0Mu z^jgaN5o&LBdcLk^9D~c!@FG&oJcrmW@Ohy+8XD#3TY7jPvIt|PafHjz{!LMxi5+hh zr`@+M^+|KgmQansq{tf*1sV@r2t$BCq^la`1gCHUCgm~xAPA#D&NGdpuHUnoQ>qz) z$XN&NwtT0zl(u!D-W!3$;D})2r=E4J*>Mm4Fl5vkYzLC_5KdY-M{qsi zQ>;!uaRg)ai#RIfZf!Uwdm!0xTt1@T@Rssn>&0_wqJ6RybB`f=n8%R#?Q{GQITOzu zu`Fd2?~rFL<&7`mY@?{?2F^|LvwVRQnpuod+`R$~76MP6^K!?x^6kOxyFFulgON>f z0A(Q+hB4pTXyWea{9^s~v0Jz?3%QQkBofgLEG zr(E@_syv4sCSGTe=^-9;kfX^V+wVzJ7OW zxkkZBiBpG#PCF};3zIfCXSY{->^OloSdGg((AvVxSu z0O(;Z6ES@yT;}tvF>e8geDvq^tvH?Sb9+kog+0XLR5!)+I;qP9Wjj%>^nW% zW$DuthTrOB#PXjr)1!F=2_(xtBU6h{wq3ia9~Im3?8TRB*9U&<)7c-0(-*%MT@PxZ zL^m+PLPJgsD<65a1(y-BI@E|?pCy(6Tjnahb{#B;470s`@XD6!It!Yg5@&K=>B# zdkH=4AoFnMi+$?_^hBB=nf&aj&Go@QzjrdgdP1N~j{tb6J^BPYu+kBC^qmQRtij^G zxxO%HqdOaf;OmkqLLn-!waU!fFFuKfODPLb|FB(8i*Y+VZW_E@OIrAFsPcS504SvF z>{cR3`5k^7_s5XK)_big(>vjGu_Y-#Q1}V^_!i;CY_g;E;udgQ|JJtl+?)?D6>pW@ z-eS;^n4^39sb%lvp?yXbv#oDUyL#_N{aMR;_3OFD^=o&wbw`)9px+YLdxPI&jLW`T zAF*5Nx#PFpczjv!ouhHz%=r|PdUgHxuIA+rN(M|CjIX@k9osj1@sA15L4U=wuy3xx z#P)7B9E4*|1%kFE`vw#8ogaB#iv@xUKbK3jh^Nan_qaT~MF!Yivs zjBm`}OEJ4zr~mP|_l>8I2F$K?UHNz-_J;V`%Rgp4Bl@4B3vWDsYh&Iwd*xGH*NqpS zQq238^w$z+Z@m0AU_P*ZW$o&yhye4L)ai z_b+Z4B#j%lea?&RUs4g1CL9dD6cqNq-fn9#<=ytBxU2t-!8waNAqMN^v;A+)1}*Nz zwyjsM_rJ3dTTHVJHfm&UE<4#;J}7M4xNLm${ho7{b9Dw^o4jwX>>IRv)YbO&O6<)K zhs2goMhw1n6y97tYHRg$w(VPY*UgVn=d7MD8GP@Zz4~9d`blDIC~n zu-&!d-Tw1o*TC1dbGueUcK$OzJMit=;I2=x?f<-3ANW2X-X&u1{Iwu6xH)cX^QEx; z*Bj%(ANS7LY}D=i{oZ@<=c7TJZ(Z%bKgJIJ^HOZHIkNN5=fc5XZ*6UV&bI&g+BNw5 z(>dE;OFP9sW(WU#8?^ni-Y)*VJ}CYpwgter04xExg$q<6fVOi%1_ZDf7i>d-IB_9+ z2+)08=ph2^C>It*fX8#;DFj417jd3|Ea4(A5>O3XR2xC!8dqX~fF9?f?-4MMxR{p& z>{~AO6G8GD7j07}39GrHO(p~WjSa;A`)Y~+KmY)QEI|Cbj=(2?nF%NzYaTV|+PNPf z(7!j*`|n67lY!FF;89h3&wcy%NB)m`<|yDK0R7j_LVy^l_7iS#ulE*$-ZSrfYY%(aF7ciD9C&eb1Z(}iv(f!LxIk`|`u32W7m^A3GC6moE>(~GCDP4TW15M9JOaCZx^%OdCcjds za4phB$)oE(*jX&@?5vj9cWf97YJc!TaisP+I}-+j{Cms!-(IADozj0l^iz?Og~$F4 z`tu;**TAP+MIhNn-m;MjAiL@a%| zQ%+=)TVmG#gak=_8vOU5FZu6>{`GR_vAq{T{|S0|#OP92(`_eY0K;eHs*XDsq=;yR z8$BCL-Zx@22mez4q)Q>%Vyu=l^kgN+|NHma|8L8j{~Y@N z_6GK^Gx}eCS0)4U0IvWJe``$fe<~oLR#ftT#f;w>Kk@%y##4HuFW)|=hHXo5?P|E2 zCE@(G|G!|ym&)pJo+n<_-mA3d@2~e0McU)$2=`yJ-`X{Ytg*(Vv2{9XE^KLBJCsUS z9dH7r3Du9*tIx+M%klF@_!Z=auPR=+ex}%dYKJmtUw-?sMghu~*b?gHV{t18Kv&pl5w=3I!ZkaN%So}P5x#jAW*qWrT@UL#xz;kcD z7JmQmADQvHw2Oxr5YYbwX8hFV%fpBozc+hVyB#?t2fo!`_HDb1e@nHRG^55<|6)Vs zFir9yaS~=^>Op80z{v)h#|qiQu+~C490j?)W5=gxPy5(iSkK>u`V=BncC4Y^!R4{f z%7e?2<*rQ)j+0rqpe$B}w1ki0sR=R=_-{oT0FVhN0ek%y?(i={37h$#e-lbLCu{KU z3MFi}qAmmex&vF@_~m~kl(6N@j zmmg@CG-P(W7U#~T%QOZ86-siNJo6O}249GcdiF0u35V|V{1r+7Sye3y1S^v=C6UGYbd4dle~@TQm&Aq^ZxWf?aTgO zoREK6ga}n-{z8y;lIh$;PRU%4y!}XZL4|I^M$fI#pC1iT(7;oR*=LMRQ^&QM;$mQ6 zY#c5_dk3L*>hR%$aWXuC2g=dS_pl3qr{sZd7ylJL;ebfiwW$0&KEn^1bjp4RU#gVl z?36b;lc`%;CCb#U4jpBH5^9kdVX{Pl^A5C!VYW`Q@UtZNa*f?oWhr-N8}mO1C0wwS zk@4$UhD0kzcEV*si^TSs=#xc80mu>}2q|gA2|_A*dD6$)jjDtZDiEpR%=S>?&?vGv zpwwAAf~1;?^D8&UVZF;@kXz3nnTOzX-f)@@tIE@g>D2o$3ilnU8pfH#jLE7P=)~Wq0^tN~* zL~G9}j$NMSucH&{cbT9mx_;DRDp`JRDiiHipwHP-$<0%t;I(Mks@|@CPw{>+Gt?LO zDzWOsN%b;mxRb0fXsQ;p+c5T22K(#%d+A=dp39hf8kyR9v@?ym9NxBOlwwwgZC>eeJUy=y_^$#<=H-(`1>+unKD=M>Agdpz`Yb@DXs1iFWXIQM3TlB$!rn)kS@ z_PEozB=1jTSukZf;J6ksJ4@Tq3LC`K{l-<+isLehr1G>^N(+9dg51O{chpsYm-qDN zgPJoP;CMualP7@DoaqTKqh-&=2@X;F@M>XjhmW|mbkdy~Y+P7BRGIG^uWhF-zABQG z_XizN?xB2L2}WME7?NJfx$v#lNB63%2Kbh$!?Paq0;IHto~%bujKVNOMiAr`W|ZnR zHO3O(l|;+EXW+_S+Zc{H$GAwJcbFanpuGr(?JBp)UaYMWkm_YvoXG_pGQ=;pR)&*W ze=q*A{>}1fQxb8sz`!$ICM+U@i@vPo+@!dnwATdNc^jGTKdK)7R$8EN@2i$*RBPo{ zqK}>t=4U1#$HC+0?(8+-p`U7cbnM(RO^Zqhs#Ej@-pbWt6eRVe>R8(0 ztc^HkAkQ?_pHFa5JdxAP$w^iUo;XMFD$qm|w(fkpsmbQ4c4g53G=R|x1qvg@n zRsp7so{=P_7TAK~Ca}B1h%)XRRdCr4t+}%Qi>~*4YvPUGeJ8yF2{oZ6^rnO&L_iI_ zBTWI}D@CLzVo*dB)JcQTLs2@0A|fgVEPz--6;Lo(0b9U^ii#a;PJVlz>)QLpIj`mq zn3-oiYpwhK+`<~=mrW5D4{{DW2Jr%}%)>|L+;uxDSP=-@WEpcXk1K{57p>CgEI?KG zfZfazpgv}a#{FZ%)a#pwf-9;B(`>b~TX$SH*J{%8Pe+7h0_v3p-l$~2Si3$J)iUEA zk<~-z5H?XxtMSdgb04XP6k)p&T2DokIy^4U#?lU+p5*^rQ>hA3H`Z0MDj4fVPf$q^ z0-c)pHTl=(6`WU(@bp93=3&Vy#=cPpY(ohR46nkVL^#oEgaByfhg2<^DEhN*bzG5H zwJ~`V^JA~)^N1%)`to5W6I;9eHp4-1*?2sUU^p1J8be#Z`<|Dn+k1|QYG(>6&Tj9@0{8n z{|v?~oe?aU$~N{*`1~B74wDj1{KiH%tWmH1arc$Q?Y^n~jbCDq*S}^cU6$l+J@%}q zYY*+wqucpYU)!AizHv%fd+%i6w~na4ZzFDBe$X@Ztuy=YyI7_EnHz!MyX*cgC2#DX zdpPy|Qvcug%+mhHuLFNve)RW4*6seMKc;?M`~CMLsB}e&+PE^P^>3NK@yde6?Uhld zf1d=US6-NG{5cWz?{oF-E3X`G|D4MH_eH35bJ|T_pQC>pk6&uBNcjCgNha%AwVIf);rbsmTsGj>e7LFk(eH)QBPY9_?=#?R+Lg_inJBvdnW&2%jqs4#7%*V>7@%ld$7u9M$T)m6JzO+w+%`(i_`# zE|zH*@edm(WgNe^$H#QvnZ-Pf{dzh*yOqY+-PHuFB&soMFQtq}`Lvr(iCD}fTs={U zB>)g%=sd(tV_Lg*y72w6q|B_H&PK%{S$dm!dJ1RMElHNy zy_MY`U3mX46~oFPLo<4fXdO5Wf)N0uW$mgvUQ6~m79;s5r~p(7-Rxa%RJC*Go}!}$ zhdfULma$gM;i9iH7EJg6Fuel^J=NCCYQ`kwqJD6j;tqdAalV;?+b(Rz^$I+F=s%CG zeG!kXTSgPxlD^I_bRA)8W4nrTXc-nL zho(ax!qq3Ob8`U{WcGjP4o7SZtjpKemK8Ov99s3GV+o-Yw6z!-%&5mj7Dt20tT)AufF3NpiY-NsR_l``08#)Pl)$x@%YF?&BU&r( z%Hqmn-64Di!rc^n+`Il(Q>Z5sy+W-JyTFSnm~s$hmB3SrMIUhkwu#Vw1fY76kOm%K z-&<=)KKg#3`tyM!n7PV)(*lJp1XYzi2OdDun3$GWe49ksUA#uUw{c%`jc;{n(ms>! z<>dA3W8AxX@x5H5aJ3y6B31~C`n0>>K&J>{i)9!mNm`z6V^cSegix%0MPDk$lP8OuB|kb*Y^g$p=tb zV#KU{)9duyw{F_u-AxPKn$APWXbA)&^@0*nIkX}e6`dnyz9Bo%WlY>4(^w=RU!ZGK zd&p8a=<`OG`$Bghj3o9vd}A8T`0R0(Y`3XtRo^l$z_G=|TQAg?(4Pe>rD2it$Q}_) z0YqJ2cF5kw)nOt!#aniELzhW=nlEuK8ykzT>qX)KWcnHC03?ovj{_ijvw;7y197hT ztAk251vgQ9>i9hX*;JvzL=S@yJs-6fTSD$y*Vi$Pt-{J=Qk*CVsv<>SAs{g{q5B^C z=`xPjtgxG(T4UZczIOKoa#K`$ql}5zqs(vdg!PDYiNONc!|ge-_*f_4?7UuE! z9kxwCt^^-%6W17#d(*d&dXP}9UzuvwjEy3s%a(DbH`fKcq059-iH2ybKu2T6HDIpg z(Kq@a1&1~u7fW&e1#HD+d!4ag;_VndOW(sVvOnMwfAB zJJ2_C`8B;pd)ss0v?T3*uIt29IC)psk^;XTiv=%tx5eUH#k!+RT<(^d<}Bp+vZ0G# z$%C=;_tf*=nW4iDrW_Id!Cho@0{98C& z9_8vp{}r>(t`=fQ4FP8>X|Rw9AW@?(pklaR3~43l9B(l z0HP}2DhjYeLg?F$&KKs_m;3uquuwfg(C@Bl4-XhC5a1;utXz)wuLB7EthRmTY?D@@ zJ({c7KQJO!ZEJBAvXAJ1u-RkG$pQE^%9+idT>JUu{na2Iz=;^Tztf>>$mjh&xk99b ziAAV`u*kItA-q(ynuc)IET14bWY_KbN>ynQ;2xic4PUa6QQ!x^KxN5RCJdMz0jPC> zDH#A&%77&S==b~&LIDg^0mmSS^Ip9;7Z{XjgJr-2A;qqY(VFvx??wxn$AE@d?6p|* zxksLmAjmpdy}z?sdk+Oy^jYh+zL%2-*2vu$OHk+*VQ!kj`(p7`zz)ZK0R}+(I_K0; z!&V)Nb$@%2q7as)eFGVHafh98uMxhOF)9w;{kS6Z^^9(wO`J;RsH&)^hlb4?wC#|f zS&l`q&T%IGo|eEW$B}i3^*V8D5%}fHuG=R&6e z;`_j3RZ`p}a9I<7Yuy}rk_j~;?EIBJ>3Dy{4qUbULe1UB5GbMD)CUd~>`Hjt>N8k* zzq)nG?%XeH6{y&_e14?%VAH)#Rzou}G5|HmfIZUol9Af+QmDFdOoh3wNyCPseb9BX z$GUTtD*g-zOS9pjYL%O~XyW^2$TB2E0&!#FI>59i3&@qQKynaL6x+Xzfgg?vc{vLN z0JvST`EUlHEh$rB!U7~vz#q^f00E3E4g$!zo7zS}&>9gM84F+;sKfvW8N{B71ypDc z0|-#01o%)3!2&>(2ruUb7>g_5LFg;e>ksAKv*i9f4uFWD<4v800J@clAOk?|U!_CE zkSAoA1pzPu8%Ru;gYY_r2C1gY6=YB+00xL42SME9ambZj&^QSk3CuYJzUH^+~JJCtQyJf65%+Avy{Bi^GG61`vOZg8QZnI55#a8}K~Gk3O&sl#jp!5H zg)rnEe~f1!zC0H0Is)xk#*H&@qGi}kl2IiKH5fY;wFjE>t_KnexuFhZb|xm)H;)EI zLcGbktMT`N+t+}rBo7FaiR(ARegC!DDg#<9!2K>po2S5ih1gONhP&r{4*|6}H3p>u zJ*RMH0XF^wS|dT0h|oMSWS|IjMFv7tm{*Mmfbcj3LwKSo zf{2~~2_4vo7Le`Jv~J#R zp_#$b-_|)bX0xVXa#gq`G${RkK3ap`6SlpT00_<7e&yLu~-{^G(5PtYz%L!WFO0sIk!Jjq@_wm(zuSRmRxoeXoYen|eAkgtV7c zbdDMQcSDSt8>UR`Fazw}@OQon6E(_pw*xT7jLC`dB?(7@PV_vaw2j;a!ui!QG; zKT`%y)yl@pkKP@dS;xO?sTu0nw^(lT@01iCniDs;3L8N`4MR?-M*&vQrIi}%vI6+IU@GdY%1zigg#%uI^g?E0j!wzO4;7i zdXG~CS5(fAa4*4scW+Fznx7@_FWobMm|sg-vGJqDJ9#rte4G)k-Pfhgw^5molJbVHSObg}d~ zUelXV8duAYg(_YsWpqMciOA zlpwUOZc@nGgw|J(&+uB5_hU1a7^(m+eS5 zr53wxpsu6&JsiZaU6V|KjWNGWfIq(RF4Zf6q!5rjMdCr&B;YHzs*T`z^pe?36C+;s z$!;FoMo3VlHt)!;jINFa`l$_hCo0Yc5};O3H^w&AMAb(Qmnol9hp9c(C@a@plcT( z(vp$dfk6<1)KX+xlZ!_+0sJwXv_80xedtzq&6YLK*X}ZKb$=BKi6FVDtCEXB)wsUza zHfT|?4}C{)CCSim+#7Nz{e>zPj*8GuXW3wd)PyEM5!wY}M2Bdrf8*PI4utr}a*h5q zfq1Dj{2-NWy)1^hnI>-NBbB~b873e|LnLg(LgpSK!rfzDeec`<^6X{X@O~mnxvPK= zgxrO=+dao#QV}=Z8=FiNGcAJR7=UD%Z?hbtc8qd6>8pW-@z=L{Jty63^uY_sNk}wr5+COJ(vLU ziwyuQ9p#{_V_|!xJd7~_b5w1{Xf3laHf8zh00_t#ZU%PN912q$J=;-jnkZz$nL-5~ z4T5@c6oR*lBbAsk;LZ;~YBfVu?9IZi&xJ6zgBm@EmP%1S4T2c#GCUXzNaZnSHaG*|TD*Y* zGQ#B(8noT}t+kfxXscCTLK{C;%feuwSgcJ1`j!-O04 z8zB*k0k56&2=~{t?(NC&+xOD3dGQ}dm|Phh78u*0vWenzMGNB+A#snX6q<(4glIZ3 z+^~OJR76a-U{80!KG*HO&Nd&7xlI9oXn?w!9}H z)N#Qt@MkI@CITLQwLXYCrdH&h`$&I~Q0y%=DjGoi$pnK!g+Zv;`PTTfHrU__FN%R@5K4tG zS7C_Lsbw6UkM>NQH_$B_U?+8fs@>jYP-Hi`M?|r4r@7jypA!7^FUR9eSw@8!t*U!- zA^vlvW_zp=_J6_|0*>|}M`k{5d{Gbf9v~l1&!et&poJ%EEg8lY(cuhKR0yDVxXZwS zz}Z~3lW*7C9lxNAp^r zXg)meCJUN{*#vSuBLBo&FtMhJ&Z~-o6n!B2y$Cb@-Fihkuvw>f#AwUev&!ww_@xOj zac0D1*W|7!YUrH}qm=wr7bq_k_~6x`AV9I2vU@Y10c~7{tF^}N=7|xgt$qNE4%N__ z0WrqvA<=Xw!Z!A@X-E^aG=Q;{C-qP{V{y1X+2f?3?m5^q2B?#I2T+x08GS9R0exGq;a{tOyS{gp*$ zYAE}#(E(KTRs7|aDBTYa3cLnsBiFXWU*;7-)dwkhnuV^s@0+uXQ(7Oa>C8bJyja## zKKOCx%5Npjq6PJw4`SqGam=3p`Eg`%CT#I;!|P+7s}H*rMXewVAQY?+5<|jg z{^}|-ElXyoYfD%|taVCMr4S~<-1MYEl&$7z;iTAF=ebb#cadHwFJ=Y5MCq1w0CL4X z+Z!2U$^n>&NE*&e3$jKTu;@oTCF5oYfZT1qT-^nRAii;qmk5s<{zR>*Q)0)S+exv5 zRN9!Q!?uZ`e;gsMWW}dH&qIq;5gi4FJ`I;rOjh^8Ay;^5D?GZuP8Ke-qxD}%iUXWi zc!?{#I6kcKva+32PG}g~!GN2$C?SGWHt${I?`wT;1;53X`_G9L{rkeUf~yv9wEcPo z@5tAy7C|-lOKek?2hJ4V1l5tSyePS5jOOP;VEq`vyk+oH*1Ftf``yY`>7fA5v?rRv zSLimz5){JugFa=51Mh%?e)V#GbmR=y#}ZGJ)br6T(G;Zxe(RALSR9=$$D;5!3fL@e z0<-s{vcXXh5l%n`F3JyiSiJZIbxh7~K>85Mk=52^6&52Hazty zuqGAbbRra@GNXRXmlK!8M#nl$%|S>6O#rO5t7ajCT9D#?-(^7T3j?yCTfI4OspC0| zD`<-2@Xaw-2{7sh1hn%+vtV2n^2TR)FafcZ2|sBIHbg_FzFbKiR|g*#rIu+;(XJ}` ztb4L&U2oCN?PsnI2e0pcaVd8OZX;(IFk#Mo*mn!G@d`rn>8f2n%#FgajkqP}K;~eD zi}Ao0G{sUMz2i=)_fN|DWrVvJ<}!_V7YZz87htti7J^RLkr9DRL=4cFZ;&4Y@&IM4 zuN((sLer1){+mdv5dv-uh?V_$2dqqYMhg)1Wk0c)V`G}HNE0jKD2T5Y%=jEH0y1W> z8D8s-m{(Ev&P@oq9z){o9P1(b^|nj!>ywc>cZ+lut%;Csb`^zawCgDjYzsm}%n&qLkd zp-bl>ehLc(Xbkk)7spDC-pV^ii`|D>uiiKX0<7YMFL%Cc-5Rv>m)AA) z+ep7L&KK)jfhz$M3Fj5*rCU=r5Ef|KD;Kyek>7pq+<6RN04m7+MwXO8Gtj0yD1X-d zgy)zCb)e+2^ISl0`L?%{&#qj5b_Sd`1QQ6kI>hpKr?dI*3z1$ko;{3hjK19TSe78yOJlv7|MV^#{=A-^vS$`pexcoLB79}Py|Lf>xh z4D7v#DK9u8>2mYVr2{H*yk{H;BcGUlm=AI7YEa`FO$v_s9#D>#M)wFXYjpClrvMkB zVWc_6bYH1?I>ODH68{FYmE8GBZ@Wj;wujk^{fmE{00YjV*AuON+AZw$~OIm$?I_Tf@ zh}n64T{86$^nTRa(FDr<`IM8F`dG@#?$~5*>X;m8br#=?@Sz;T>_O1c>kke4-#5_4 zCd;b<Nr>WTy0ofvnz8z;MNBy)l5KL9u)wmj9g0gs%%?8G{-s z5>A*mOn!(e1UZsufO15HO7cvef|dfG=L&$(nYwGCmZ#Y)qvZb5|Ejv3S6f(Lz)}zu z4GZ@&nyEe=aYT1q4`vt+KeIUW(z#1?4V(V#hhQ7$wjVxhrC4AY!(cw~(T+)mM? zK;wnB>NG_B*h%zHUP2aL8prY&Z&F%8>4;0f!zJFckQf?okOqrjvflX=M?U8O6lkcp zshmWO9K&OQ5;Fprxrdj|;GPw;qCmuFCx||!n8j3|1j{l9mDFC``r=k`gs!$*d9rJ! zM==QTcS()-0=jD~SCyq=-PG;52w(pjrJY=`d{i%frMrreqRDSHo8h>TSpXeYFTp!| zFI{>Agwt|Q3Rmy*g!MhSg!#-1_AYBQR(zU{aAB@?8><*yU+*P`0ds-vXuyqN^0dbK zX%i?jczN`Xeu>TuH#pf|hlFsaK-b>lb&Krv0ozDZzTB*-H~v7!y)D^7kFz+VNi()9B>?LfJampK))zIWp~ z+uXD3RBh?p51YW!PJuP?so1Q4CW6(tk=kQ9RMdI+ixOx zRt^L8d5FHu9m(Y*BLG;67E0gCK-62}&nI3WmC?NM(~*@Nh9q>(9L>@HOg-m77@w zk>!XY!Nn9;4w27EWN=hyP=bA~1-&r74=x))e>`c5RbNIxWO+J^Zq6hO%{vtH_9~ctP%fP7S}=wbqT`jg9`c-s2u|aQQb!qxo8q{ zy~c)T4f(+Wo?8~gpc);L535+U-dU`LeQvZ!0OXVY<*69E}}Fl?5FG0Jge4(3uXqBac!TC%VyK zJ^;sy#=WtEWg4LAv|PNC@vX7lh#DnzVdGvcr3K~*Ab6A=%j)C9XhNu6FVCYHQ1&f* z@T{^8D4u%KG0R7=h^K!l2I9hhC8B&>vF zWIMr15(|XT5SrC&8#@HKnMF!AlD}jd4s10EsyqsbT;wjqL1-W^{d`Qk{N_(Tb>Eaz zUI+^)m2=S1;e(!C?Lkd$s7<-}kQJ|asyj!SteJlzX%MccqRFTiVXN9#IeZ;1KtKjn zyt=UKrUl-rBASVcVbEHWTj~}8X=joLEL7lX7@)P_;(P)%l}AZVmEJQ$K$~+TyVwb6 z3|=dYuDprwrn0ejF7;k|GO-#7R~2oNukX&eVlFxRsWf4_jE#Y?!J}ht1?2~o34B0i z`k`o5b9`JyAGNsMKd$zI&&w1~xS}m~7smDVy=vpc#y#~B^LlQ`4zoqd2{(Pj3?b=u z=AyvqUKfaKnt0)+RnLp~aZIW; zLuk>KAqjGu`#Z!on>0_|;0W`LkJ?aLDo-eSv_PBXMRcuD~S~G{f zi_w@QDT^_x(%1>BQ9{K{BSt2JYca8X$1SVb$FA-`65-2Oh2G`bAe#+cn#!!W{w)z3Ko28wr zzwLCdGGH>Bac4kv{Exoh>PbVBLJKTS4DIurGPz}JDFO`I^i=(kxehB9*LP^0vs=->sXMrG zr-(MP145(y-~>Yx6{RndfH0+Ee|`EgzUH2VTbs$5jM%Z6K* zqBQL?p+LL<2E%&dD88lv!-F`>Lc8>G5dRdPQRPBXQ0CfTjvn>QH{pUup?1j)(4n}mb zJ@p|Hu&~n+q5u`?$Y}F%vRpr|VKizqV6S&+uLt`PJGZ82_gX=_Op_78dE`&Sk`ZNmQX&Q~# z(`&8wGtWJ|y%H*GbmwXX`{ol`#29|isUvi+9oQ_FnlRX?$zKjSjWBnb(Cw)y6~~A% zMwjdpwYHwM1{6Sj)#G<9AWTS6{Nn*F3z41il6dQh)W9`;Vnj#*aShrGLY#wBnA42d zg9R@wFS@bJ?iILiF(f|WQb1H(7c8qjh_}^f3f=bFPW3IIApdFBC|CF=hJXaaEd-rk ze8dU7_~HX^F7J*fT@f>_e3FJhqUHO>_NOBll) zqy;%%E*1i)BoBn9a##^(;$i#eL5v(_f;N=yUmpa4>jEHIHCB4WFaX)o%bwn*4_UR^|#9F4)C8Q{WEPj+_` zfYTkoU=yUAp-+^9r}CdZi%{TLU0A@ezCZn$c@ieM5abfGN&Dwv)b$mGlFbzui+A zpXGZa94!a1*$?}n{2>F+%k$p@Rp{6;M(U#?Nc5R_WJQ2wtAS)gzy7g3kKbFiYGtSR ztDM{Ocs^~LHFPf6K^^9)xFvd+_TeG@8t={pK&yQMUOwpV^GVKeszB$&-?Kx(@CHqUlvzb0QC?So?sXG(%tMRhC z{_5aES8W&e;Yjq;Yxt}d^@+T*_ND1hV|{`nniS6WPUR#zh&|4{cp4oXY^iTxM%@<^IFuyy@;sQGqd z#rV$X1DRq{i?Et=@q8k-$9chJ=*Ptv^yeatJo{J9na3+u6CpwK#D?X*{kdsW2M2~~ zEe(RSZ%Fa0K+@|zTz2VdD?1Zf#cqd?zwqOM1t zZp56)zD#eOzr6q2hsxdIAMV`Eod5G=*Kkk&+%>g`Ra?GAyL^iICKzp`RBxcp?XncY zYr;&<;7XQ|muenLQeq@e^e*g+LZ|DY_T*I@kfJgvs8@}z^Q9yE@;;_$V_rS|kg^qP z9QpO-Q`~IfhVeSg9SyrR(dehKl+dx>6&> z*&Ue>?gmhA`b6E{{tWs6dtv*dgH2R)9Oe06&s{-La+!?ko@XB}O7+Vh^*x#CU8g^N zR>*^E8_ki`j2n91#~Rma1sqzyQDb=XRqy*k`>bbhihAWwj+;JxYOMJ*r|H>-h`A?+ zG)d~m4Z3yUeewF6DK9+;cxW|sYXx*hvl-!Yj@y_LbT=_lNY!0{`#yge@YGh3Y+rk? zXV)`}%oc~nr`ads`wkf>l^CpTG;nNd>50>pkf?Q&8UdTMFCW^cyQbs$&yfXp3VBuA z7F$oM`oH*fPsud(hN9y6pR1xt`p@~66x+mS)UA{?NQ&K-=T@5y(UA(%KbzO38G2>5 zdL1_O&TaKBNibe$d26QQli70n<#XPNR=;b8{-dq_Q-%Q#S_7W8?jP1S(ANt5-nwya zoBTfde?`{H#q$3jPOSf9%uA>63jY5iYb!Kje2H1*M=#w;bE?eRZp?=_PB*m z+zdoRwn-#{-5CM&d4Z>e)WuG;xM75#6vMmT$K-nZ?BNT(mU zN`R;%T_6s!I2rqP7nU43GNtorL3g#xoTN5Ju}^Lh6=r*q{vL-eocv#rwZBs|nad&> zYj1p49qD;ZeQZS@j)&Emp%HC#Z>zB$%0|GwuS>M_b%xy`@v>;5u5QeBF+fUt+=bFk z5)aJofd;VhDD3EDJ(mP2L{&R?Dcxf?K7xNkH;v{((8$h^j^<}GzHRi6CdKn9X=Cl` zx=c#jN>P}2Afawz)slm8uAYd=pJR%H&yHmci6_K#rvAG5-8Qbp;|rq7crw-pm; z(=I69(;sAw&*NS&1)UdpLu&i!)FBMdI)jLFU(Py4a2v_sQT)-eJxCRQi7??zL_O$M zc2?aKtraSwz^i5czc!L{mKYF4OXGZVqS6P-dqr3=1+8h-uBPh}y4>}E>`ov)+(1CYm?RF8Md)u6rV6xvRM26sng6W6ihi~w$1Ja@S<@q&_8U7{9;Rre@)nIBJce#q9DAbyWiG_*G*5RMac>VfWUX%Xf@ zs)iPaWBiRo^W0R|#{--eW0N1BB6OVGS!ag+OQ0)UFX{Wjn8a|m?T$d(I4FvC9$g!;rM^*EX z>ocsl2A0%E^6;06{}Pqr-E=Q*C0(&4@pA38iMW#UFe>+E`PInH1eO;eBw*fmW@odh zm(N5<;0t|~3Kvx#gEeHNm=K=vc*9Cq<4Up%L*+T-)d%^-PIPAPaf|0}j?B@w%n^d3 zAzHybq3qtKpv?W;qa-W0{?l*DxTsT)?wX5Hc6csg=M_9&?K#rxAFNzPaIe2$I7Y1Z zub925q!nd~4xRhbxT3n99O@$@tm^vA+`}1~`w)Aoa5;sNO5uPZm+WcZ%dH3MU%B#q z1{N=rBgX9bP~nN@cp}6L)@AUZ)dxZ(4Kz5}>L$AL12{~U-?EDiF&;YQQBqO=s)&<) zhkQ=DZ{ynWfdTY-*l^YaBHcJObbZyty*ByNM!?ERU~mk-ojdwb-T-_M2%Uh_@XEK( z7+C_;Fz?&3o`vnP@t!dDXDQs7G~xD`;&DiA72RZP5%vkpqub1TWHD9Mw_LELMg&ky zq;AIwwo|E#t6{~a^Nc_V+QITYT&bXDi))HNMD@sW_V9hRKvCCU%C&SwH@jqAxNt-Y zC^_!!NKNvKx+4<>K(@@qHLtiKSyC5|PqU9V<0@GH{$nZpb(tD>6mvd_UDhX@0*F(z zh7z*T-rK9(Y{;%0oQKc0ythEOLWpiMW{|**8`Q;NFD(t$v(_(GkfC6-U=E_1)?o+%UfiH%Uy@a42PzYj)};TCUCn^@QOUe*+c>HfA;ZLO^HpIF-Eb9uGO8@p(qqn)G8TfwMCRlOPLbXBTP02=@^4+9 z=l03;VSeY2zV?6ZcpToYkbdirp|8db%RKv<)D!jZ{TE%eG~>2x9RmlG9_1_kk_bvJ zw2u6Em1-#qx z=+}eZsf(TaC*N)T{p+Dv@scPnU@2bf_sm$}rS7W9r9`LSvyzfa7fuGe-x2kDZf5Gz zrJn!g#CjxE?CHG`@L^Bg@5hUQJ(nL&e#q?q{baeM=gRAVj|U(9e!4Q%bM43E$HTvW zKLeC{#i+n#w$>jhdSmaP#?*4I)1P@lsoaUBxHJ>=X93wx7T!3q*yM#Z)CM!|@PzbM@cD&q%}?*-ZKV30MlV+HK5#PlTJ?QOq@AM! ze>`bxL(7k1TkGS2cM3~9>s4;N)*JZcC$FE4b{Y z0$=-h_0Pt@zD~V#e!oud*JHK!nbqKb87WHH&+=NTs&^ybs=vqIRMPwETy^yw_Zi>g zhoZ`B;AD}7M)v)4TYn5ZGx`+imA&|WtK-|)Ckujezs#Nl9&9PaEGB2EgEzNno>GuAaom)RQqyBu{cRQyu-0SV*jj|sb&t;D9 z$A5X}kTn{TA}P{p{^ynmH@%4J7_xnNGbqW%O*GcYg2~PrDtgA>|cE@VajB6SHzFl5K2A1+5VU zN%Vqd(}VFdJja>rn*E1ZGr7m;1+`;*b~UG@IzMuzfRkKMUk&zD7t$i~y~hg1+6%9H zbB-k!Z0H4}#)@uv?^{CUTW9f}yk__PD7xcaEUhk3C6!#9+5ZAv^xCv2x3~Cydr_Tf z)<-mdp}GW;RroclbfmhZW?t^fDDGV>*}^Zqzdv6@;{I(fn5P$=Yd`}9ymAE%Yx2`RVCLguxIhquC$oxR1w%M`gGlFt7x4=vAu)Po&c{I(Z zD9fxe>*vwCGgUFjdfQd84c zoseInXIkAiopo)osvJ{&lV5PdyX@vz;b(ezyIDnb%2DN0g4j33xwFT{U)SFDuGJbZ z(B$WvV)Cx16y+yZd9_zv{!v^;KGut=yGkmqB{!@$Ylu8hmpWEHbRd7Zx6)6euHLLr zPKxi9(JSH?>ypUjtLcq%$(8^0@o)WXte7dB@M%2lll?up@=kK&*}j@z)%9xmjg2Xs z=2J)i^R6V%ru^7nb?;3nZcZ4LU%xxQL5*B$HQoT7JO1@e@RoC$j7xya*x-V{{+D6oB76OmF9m~y;n>{$QLeB-x5 z&x5Ti2O6C*M}MQsQu>;(=B=&chavezd)r&2KaV+G&eb`<`hqF;oo#GC)v#fr+`pp$ z@~iosSraj$oW7;5qo&1Z4vaf^l#x>VCZ%QXmi)(kg4G>0t0sirZ(Dz-l)Ahru?ero z=QsJtG#f1^N)Gi^4$dAk+|hC^vQc+SEi=E}Z%4K1T=g${i$My%9(yu%OMABO(d_Y4 z9hYmu%*&&E%TGAAAMrVT=Tw8Nx;C`$2rnNz@RQeh>Tp3{8DUE!c873dWXqFZDIgPSo8fXnizy=3#At&W?r|$Ae0aoo>H6 zDdx><&}T*7XR5cfKJ925a%`V3IOEsRe*ZvwcP;Pb<<7UiDwO+9nk64S+HvmWj^o>s zJKM~<1`br_JZwwu+dFkxlyR^{is7Y?*XEx*%h+-9b1(R2M;XGR9sTI+sL$~&ruB#3 zRt@`9M$7WMlDD+%{!!2MIg6?-V*TnK_o^~4=*sXp_3~ggzoWUQqtn%|t8F6BZY9sx zRQa-RQD7EGuM=LRA6`7wU7A;Ib2IDXL(zM3sb$i!nUma5Y^O{6@y{v6%MZ&O^NBY* zMB0AmPPBveS(yVU(zd?Rn2B?Le>DYao$=2;{;}_TMEb>q_S~HDg94}XO%r7{7AHsY zD)XbdHf2}mz3D72=r|C5F*Bl%;g>PG= zqSv+Me%Y6un#y)$XVHBw<`woH`O#An-Xy%)*NW?{&K7R`)oU7edBTqu+~2<9+kQ{$ z3O=eg_7dh=8SYBu&pt4#iRM_d+w`*a@4dABSEEb>tLDK`$eO=$pC5~TRRVUYV-k zF_D{ujbEMRh-Pi`Ca7P_|6^|P#b@* zGf=%TNX`z*LvE6V0iN1wQXr3#?N61EqlGM#rVi=YlMV72AWH9FI*R~{6@LoRW}{s_ zg4Hr;2tD?4dhWiu`_-W+I=>PXhe>ha5K^Plp(^_`C>mO+^-QQ>gW5|t*|pw1LG58z zo0fuIhL90VZ-2$bn-uYmMO!Yu&Lt!(8$DFZrcuYp{xoa#aQ`K$p#r>H2>OKy91yq- z+fOQ!v*k^5G){M*QdLp z!o>p!7mab5m%Da!G?iPew&(>9_Fo?07K^*N8Zf&TRG2HM7B;xSU%NJ8>j6@YOK>Xv zyXS?hePVar-n2TysI2i#tpA6xxA1Dhao>hv8*KDOj~tyMC9Q5SLIk8F1d);u#El#w zY$G=bZPixUT!Uui%R=fPQ3D zm7F*0GA$KgIEUeN^YER}uQO3t0s$RVv;;PkFfNG$<{Et*9RK{bMOC zgdKivxOBSBt{U&<4j(eT>_39FkIS6;kqrxkW1m7~oDt96*gxSt+qA=cecQ+79Zgw5 z`+Miuf@hJpuV6KPeLk(QVaH;Oq<=ZbYP_(G;6uh3wNd1j7THUKL}?U|JCsM6?xEnk7X8YuRW&@o#(PeoPo z6p?X z_l49qWZQl?l<*pB&;8~$cSg)QLWheN&Lhr;o>`Ds8-#Wu&|bhZREeH8pj7x|a-b;8 zZ%8{?i$;Mq4Bg>I9VvMP_`NP7YG8p4&1 zL!?AZG|hF{X?Px9J`3>YNMdnn%PDX)<;!v?R2P9pmnuV;P0X2Vw>PC$V>mcG?+u?a zmJMKUf5m$|l%(XT!5&};fkOeZ8d76KiJC$G>cdllRy)YGB*j)@L zD3EJ`5d@{SsXq~Bzo4aOtHmCLTtmhun;g=G2#9^efa6!|O^uki#9Vwz z-MK=f;8dnu8ZAW&!Y|D^NWme#+$OAeNpYJLA~;d3BJ+XA=q{Vc=EMBJQf1T3wgz#z z&%#B2_H21H(ai;c(E6;)86j91YD@QY2p~k`i>iGiZ*|hiW-l2?e~zz z-kqOWJ;0?&r^u*tv8I!KfEsK6WL6CgBghxc{v6U~{k=D_*U-koLd|ki&CaW2 z%Y*WjQyy8zkGQVQ-RsHeUim_TQtlkI9lEFNI&}y9Xib!Bb=`RyC9}*T=VH0^hFhg1 z!5??I?UWJY_2tqj`>`#JRO+n}Y7qrx{gjrfyDsulSZ4D(s9wGkETM)nIGJDgSepY;&(x=*hO3;m^6h zr@Vsoj^)3ne8b;TmH#^BO+>~TPKC4lcL_5yQ#6wRv+Mr=?f(`gJ1(LSxhkrHmK7Fu zo}_dEHjj~#_M7Gec8nD>(2t#)AaJVLeWIiOaf*0A8*HuPEt18+COnvmCqr3vt~RU? zz|*B#80kM};rDJmYcxErjPK*+%xvP!0@U|uX4PqFe0v4EnQ&dyulr?w@Y6N=1Z1IY5Xs~yMjYc;-EA{Z0Akz6x#_c>jYW*EFgb2R(BuMSX%1tY z_{8;#6ioW(Eq;6SZce2|EOb9R>_4DtYC^(#L10)6HW})SRTP66Xi&g>ays-;Ca!2I z(w0>%(w68S-BBYa8m-MQlqL88)@o|Cfqia2@sr&NC7_pU&Sp4AGKUM!feXUMzv7f4 ztSV4%>SH}Avch<@upoB|XTa_LVolUVFC3c8Yw13M1(5#SH=O0>Sz{~kqLaI7Jc>nz zSZo=1n(N}O-JuFJL-3MdLYaFQ7K_Xi^VizD=ApN{xC@*G;NfG5cY;+_ZeFr#Lw#V2 zh6c{v(f*xJfZ7htJ9k`&k+lfsZ5SaUf-P{jT-H7Afkor&C?tE%0R*}J%V^D;6nCEa z8pJoEY>~&=5V@R99y<|tN@RzVrbRH6Kdp8w!L4^4>uY|v>V9^|L!#&5vVPqd6R(cRI zI=_6udZbHrgdi@Ac;?K!2oirm7`h_j#Q3Z16B?0T%$q4aoFmLBNMgDRTxfix}Oq?W$FLHxQ z@2Jz;%HCPMVAgd^V-fd6?SiE5laodx-BuYvW6X^@e6#wX%7unF^Bej#*3nZz62Mue zlRI`ZF|o8E?j)Lm$WIn9tp$HLpi{3@!OU~h{>@psM5pR^isfZhH>cl<{=L%y1jM~H z3O$&A5XXfHuUA<6D??rL)k9iN`~b+WHjQti1u6>he?`B9&$(Z#qyw&#}-FsoUV*`)R%zf@UR- z1h`K34*&Th-ItA)*8r{7qWC>Q%*S>l(YWQ#rZ$qM(}#@WE7roD+x^fhN&40Am{>QZ zAt#&LvgdxSH}cNcWbS%80Q5KPp?WzIx0F;RL+dQ_3S@Pxq|O*jJcdbr=PBFVV$82)~oU8xkDz9tXDv!txe8C zNFZ+{V59ZgtD5vpM?7>4s3SPPtwN_RxHJp_jA#f98VO9{tP!s;;A&xw&OmVH-ol9R z@;C=_h{XKN*1V^#!{z$kb05k24av{rqD5t+&O%ItYcoM#ZkZbt@ZPw}AtMTd$afY# zOy6!YNpQ7bEn7N~Mz6MZo21-ANq!StaGyk^c;U<3%a^;DqT&)TJ3NO`2x(cg1F&?ctCTwi@{L@~4Pw z|C3}V>$gGfUo@}D{CN;!+;Nrw@Q;#SYms#3Jc0ow7aQZTZHX)gp3H7N@*qEq*>uiQ zV?d7#Ul%4t&T5sToDqI?;(Ej4oeqdlFwPK3(nvMzm0y({`Emo`--n-vHoK} zMmbh~+#9uXuJkD%53Pz~BbYX6>9N`xKO+Eh~)A*{+{9FbU^ZlGcri9j>5L?sK zS{mo~3QtWp3IK$aPF;&vzp0YBA@SIC4*uex{&(vZO6Vs-CSLoN*7Y5ygwpYxB8%oX zs_&H2MW(8z?>Afw-m!8%oJs%lHR+?0gubqh(KRU&33>WHz_gy@;4VNr`-S~9~&*#eZx)ZU-v;8 z8ol1_>a3f4u;xJI?Rnpcqa1Pp|S75<*Vy{9MM~; zI^oOdAshWKMLvcV_ZV|~9GWkb?iGz2Iij<1VxL~^SDw3`*!bW_UyH=v-RuY6M{11r zve&!a|F8D4gW{Q^w}$i|v)K>7G@kqY&W-+aapvK-?wQ{oW9h$WJdds(j-UH;P)^ zi03}ULxl;vdIWwKf*_6{TtX1(A&4y!Bn}A(VWPAiQPzbhk0UDLgp^9+{<}XG$>hoO zg6Y`*Ft8(M0`0$~4hV;Fg++TUW3oai5@E5yo|%CVj}$I#Cr;!j!j%U#82?{oJ&>i< z8He>p+}ewfTdw)cS$qPze|(U#^5?1y_XhRDSh+b$&2OF(Q*uVrEuS~POe#MJNg z2pi8ME)}>kmtJgHhs*l{E-`6^>7F_1wqdxu;mROI>+(%vx%LW;vTkNUD)pZXM*va+ zr`S0s2YWlmGq)b{FsPVJfPPT5ZZB;I0m_m(&RXXYXbCya6LxO696OP=x;3wt*0R5cN8qm+yVin!(GY41 zZWV8li2E?-UB6tKJe~qN!c%4&x|s}-+K_sg2O z;BX-}V^%L+hQ+$fcGdKf>-I+q>vc{~iuol5A%no8R#CK=0G{w#1XCD}&yzJu_>#x* z5nZUq-Cl|s5L-dl$U_WO;UPkUT*)Ra#;Z`0D5%tB%eEP|FXF3t@$KfzKET9qTkz4( zY*8eKHp>isq?+$u!cI&+G%5$=eDADVxV>La?}@KAnCH5tVgT>A-<>@Ye-BCdz8M(` zPQgy!vBL_b|&X-6oM#N!`EiFrb_y}_!&n)`<(O3`u>wM9LGTnsJ-5i>+) z@`)8Up5hfP=XDdyzb`8QYhNx~FCs;*^-i&AYS6rROJMwc$tK;s7mYCo z4T&yL9Fd9LL+7w?Vg8m4M-VTjjHHp2B(MH;|22G@mC{e!l++ZPY#CT6do+(l{7${V z^UPoRe&H>^3%(K966W4hMQmDm(3;uD>ORG%jeub20y4CqkR7_pop%eF&F=y`Zfq*= zjV335x!jf@`;O?i2n-*3F%HX7I<6SU_1Th)RNM6e`B5eo4>a!!4XNkE-SldfM;GSq z(m=dY0!wdk(3kUU$Ca+Y3qv*K)Vw~%ckROk=~$9?A(N?P9XsP7m{Ar~_m-b%Ob@`& zNh&lv3w@q^r33*I+z6H>BM<<8tvaP1FK?ldA&HRBB4Vof%;TS5S{r{YVb`v`dfA&< z`p_PgR+$7b15Hc*5Co$%vzX(UhEtQ&+k-sF-t}iJ_Z1uY1<42$P#NNx@~+gd?eRsk zr|zQ}@=zNO3RhH*$xE{#pia+fGnXP*w$O;&dVv;j71|D;E^MTAU zaVPm$1L}$Fms`&)tRnKdtTuIS@K0GM{U~`W|28||*JO^RTbYq&-Ki_S=Zy38p)afI z@3+=-EEK%g$*tQUZZ}&kXT)AuHam8|GqcR{1@Tzf(YB6Hu*G$nXx_jeGe;m>8~(92 z4IsJ6$$p}ZC^SW5b>_Ol?MIFgH(G^g2@eTQx25P#t^Rc_Ah_Gf=2mOG<%c7RHrP}p z1eCppF|FmdZ|be;BmC@el-`6*bG7mjeJ?C+l%7zb=ou!>de!Anm z)4?{K6`R5S+FQLdC!IPASe|U1O3F{ZdTq)}<`&ic*2iDL=(rFGSi1Wq1z=V9O%6$?P;1IhwJQmg-dp+1D ztaM30KcZE@J20T%aB1q#i8{O6AKZMFSK7wDw|}@A5+Q21aQ{?PbE(nC^Q4yx9Xb)! zg_Z9UV&U^gV<+y&aEHXC_m91Ah#KTMxtBWPwkX#9t*=~lH~D7&D$`6tPmA-bfGfpf zQ6DW`sU$kx1^#&>7s#n<1a?##TxhPm|5nra_ncj=eSTL| z;mQ;(Jm`74>f(r=D>HXR0mlR4lSMBc0an2lr^=SFyCw{`Bt)(ZqbRC}wyD4-ntHP8 zSVi_=6aCp;8&qDc^sc8s+zEe)jJ>=f_W)m}fpx>p+d+wLkKX{JgEV~C+>iDXHk zT;w%Nh~R95uER8O^%*ml#F7k)^6c+&9uct%=5+KuFpuO*nLk(Qm$81?2%`XqGZWuG zno>yO+chz3aQDEd3IagNj#6X)OJKxDrN{_JMmWjdVNVJo>2HO#0~|89gYDkN?iDDszc$U5+2J2W$#@oH?8sju4}f72 zT|k(s<8ul>QC-R+$!ZoU%@>QOK9%@CWu;i{kE^`g?z^aGr&U*yNTPc4y+C&vF$LL# zAGhWaFz9P^fh~F$Q(+zpt;ze}UyZ3Hqb)yK$|4kNn#1k{b@QgHGh`(Itd5~7`EHkp zUI@1Z_|j+TgJb2S>wdLJ@tj=q5UUN8FN?->vw(^5pdAq>r-V0$*QXUbu%vw4ueBV~ zz_NBkcu{18@k~7e+$mO4P+(!rJ#S05^LQ}+3HOMpn1brIM-STv+Ksa_yy#M5$Sq%p zkbWhMgm%ZyQ90H{80JVb3vB>J0P_6&xx02ypcs0MeOy4fos!rR4BZ(MxS>HAYiGP^ zOhfLnua-cg;*NM+*&}A(m&;q1^0KV+jtmtrH;)K(MYJ;u3B*B12RVp}${u1nN{SK2 zXV`}BguD0NcRUYu8YMUU$Y5Kiq^PHVj|Fw97q zJ`%<%bUge`TK4s$W=??AcUOLe@yQgvUamg}%o6bEmG2&iAy35-dJP$&Yj*g%?zBq9qbD4o_2+M)W3wA*e7`rF;XeDQ~vdL1*4L zR7V?E(0B7_;uw=!msw`P0vm*d3DwEbxsZjyW;WsM^?SyKFYTC)XXaJnrZ-NULXWHJ zqfGX*iU&475I78>L)MOma52Yg=2-Ys{%Jhe>N%%6+#xl?z)w%7f`@y}60%OW7-2o8 z2s&?6;_fU0Xf>j7${3}uNenD>PXGRt;%aZP6H3%BQ*8^>_ev`+Ap%d){1E&b#{5Cw zc}5576(8J8Lza9y4gF)1bk3(3feDkuD{WXSVQXqV7jy2!Swrcb>z2kvrzZ-xeU$Q1 zU_}f`3B50AhXKMSs-Zll$pCSDDKeO`UmpWY`1R|x;&0taMdd+Ax9j9-Zy#;7KBWjx z$1^Lu9jBLEZx{9~bJaWLomF9dBfa8R7~Ui9j=A+$U3yWy?gIV50CW z(#c^@-r9Z$WE+zKP{1eh@kWCYmPvYXb+d;C&}74a)mWY3M3yBNdVe8pm0iyh6|M=e z)TK7rC@U2%C$-Hir&jAJ@)o}kMlvf~wcGO#G7H1iiBB~kJZdfC9wN*s*E_gjAqd(6 z%~qKm%IqM`;>9qxlrHZxbMm5|p01scEIMcf68U#dxefzhJ^nlaOgG$ChFG7tw*Ya9 z{+2lWv|-!*oS2woFo#PBswu}rVn&+Zvj72=D%rK2Pfj&z@fIPFAJc7CQ}3T;zmo*EO#Kp>M`v0) zcYf%cZP(nLd`CIOV=3{Q&n^BavNv*YU+v|;=p)0{C9wX@p}$*8!1?J||H~7) z{eu}F&Rq}6tyQg$OvyYJ)Bj%e&;wej?W%UCC)DXj*FKShsN!+zv*KQS6lU_6T3_hF z5?14T-c75IXBVmb>8iN}Pwrn!RGMTOws{fRR9T1CdFk$cec{@>uWfPH{EI;5uxm>v zTgbZUtE}bx4+mP@RhG8hfv0tP3wAS7Mt)pWur6i zfMWl!E-xy0Q&FpX{1U5HVLzux$~PR<{cT77Gb`RbdsjNuahI`js_L&wiXOLm*j{jh zbIaiK;l%kW&GP*?C*oc)by@uKjZdm?^OJgOID0!QnJc{%&k0V2k7<>4b8gFR-F@ch z__>5`C|FK*B#*XAZtnZJSyIz%lX*u<$n151^MrH zKd%H`dJkWErxsQB?Dxkn@-eRzQ!F}anzlt8&@(kD-!cV$fAZ7n9VzXKs+Z#QCt3ZX zve{gFw0`W$=0U6iTa+GU!DTPmyo|>EJ-o*&^iv$^Xo*juNbi!<^g(v_(PmhU-ju)1 z=dzba>za>xA4jN#RQ(#*I?d?FPmuU8D~K^89Y4<`@-KIYfF9nhru<)4kiYr}9pmmr zXqOr0Mi@HnS!gn*;|-ed%v#Al6UDsXKYXEPkf}UIP^0G%F#aS@KzN&pe`^Y^ryZ|+ z9b(U%STFQg_BPdX)1gUtU6_5+Q})GOh@mez(C)2qsf=sIl50C7y6(X<#>LU)nS9c@ zkM;J;{bu98NejaIQ?*r!m1Ub(SiL8!?!Vk&(uD=U{HAx!CSLC~lQ?mU*~YPgigp}OKLgZlJOvM{|^vV+&)qf^3q?0>xQ*c_Tq6wL1!4K=mfu z#+v~}Uas|vdt$XL(0^XG9GyGu4?D+c1@14LWW9!H)( z#Xu!Q3f;{X({nvz=-nu>u4ZRDl9HetePjwq7HC?JuwGHL0` zSAi3@1O(jM^SU8K8q|J#WHkjjdvO#miiHwNFO^EsFSjF)rHzh_rA4pcMWuArHSbmR<#(Am&1XB52b!O&Y5fFXv5A$3=Uz4 zn7IqK0vjG*Hy1y=pC1rV-##JPbm>=t==5I}!NR6#%AzWK@Zu{W+G(~p2@ssq<|qAE z5aC3JfY0n(RD&z`&(4R4`Y!}Oi5$fOP_3Nm+anZ7yQt@4!%?EqXwv9f>XVeeUG z-zbw@KUJ6fW)8t)v@K~N;bO`Tp@fyRz)d6;Pw;E*Nva=-|J*aa10s&a(P_hMI*R?= zr$_(*&C%x2v)n6UGwJ-VaW)aBzO@=ka$}23!lBl%2qO%gB*rGzqkiTOe_Pt6W^Y*X zS7Z(b4c+<)7OhyUecX#mlcOebjxo$6kf8W#A0S-S&&(xt&OJekni8mKz|!%wzPDR}N@T5X|bJID(Q_BK)?8OAU=*fkW zIslPIW0mHeF_3|+fl{03c<|GJJoZ`wGU@~i!P_B}1xq`9fbTOzg!m0PI z4b?e$f=NSF%BvW#ruywP)Dc+B+cw8g-HSbBaY~MK9w5QV$}LBvwVBDduz&E=3T~R{ zlE~vTf;QCFPJmb;mmQhT4jbfDPF`xo#HghP z+Q{>jh5VURY%i9!XxmZ=c9aw3O_RlV&nP3_9tysCOxP$NCaNg4q*Uj7n%^2(zkGef z&YRA5XemCG2 zZ1p(ox1Yk!y%iY>e**g2D~~|F_YzU3!!#sC8wf2P!k3%^2(!bDdqM=ELD7=)eAxY?VusViv2C zSxTyYA-l6u4bwLO0xO_l7KPbT;i!rw?7Wv1tAQED7j^UPSQ4l6*cYWXK(aWFR-q3h znVFO2HJZtq&2yaD#alm~ ztZ;65`A|gT#S}u{*PU>((}@=N{SBLr!iG~TlY-5~RF+F7Vq&>ECamIzo_4)1vd%BX z2wHW_U>;Y&Jgput3M)5E(0|1fPEDEM)(QFOzLVMw+&9FmK0nPC9oPU903wbfea-laHil3b zqs5WH$}LXwLyKV%L*PVC%db4Z*$+|>iGYoH0b#5oK2h|zjh%#0z@Q#c^tJ$F=ng&D zZZ7ugwjvbhJ}BkrL$tCZ0B%}S%t`)Mb_)SU5K2-YXCP`g!nt4&BzCu*G6q*DK%gTk zey=lJp?f1E;#Sm1TKS6)vjhNOj^VsCqGF8C-yhYf4Kp-SSo#ld@Den(+QW|4_|hmu zfQda8BBBSw*7`Ah{KplF1+b{{(orCZ_p6@ogH$00JX{L=a2nEik1`@raEb)G$^H5s z820A_a+ghA{v;+#nR=sGV1B<%P}k7|Wjnk*@U;N?`JrRLqeV87h9?-p)#&V1Wew43{`drlJ;?I3DSWV)x>f6FR0mwd(HR z1#1B^mHJ~+w5O%`sKm2jR1xmF3tDju9xijCM$z%fjs-m34nk2&CIwpzZz zs~VTZFi1bD!0G(w`2hL}`}5X}AIdmRJY&$o+Fs_%c^imniW&uQKV2Pe6ux(-4{*K) z1LTkU@sTasv3>8fTpQi45fyds!zI=vp1Sq~vNM`l(tlp=^QGfQVPj8t7DiuDGtAAH z1>!awpK%a{2|_IDZ@+w=(adgWF~!a2ZI6!zrMp7(~eq#0s09KjcGKfIone|VoxUx8bg$9I)9MxCSNrFV+K!%N; z=ODSj{&U|8^{YJb2SwaWs~vgsXkTqbpfr@31P$@Mw)^=7f3?5Jqx!q23znX)i}4Pp zGBqQhiD!S(4Ls{G5qSI`2lDwtBwyhyBTu{t6m!`-mEw}&;zSUTSVL0eqE*Now;oIM zAd<{hkq~h=s^`-lq$Q+4X7zUzpa>~8*$c?cVf83tXHHf?i^z5=2U@ugPgL4N*r*>MEu!8 z`>47nL4LQ-gxBplI7Wp z`Y6rwKsak94Wt!!QJ7AYa>-;eJDDmtp0@~5*JfBnU2MQM2-9r-Y}Z6zPWqo!*6r2N zhPhaA?wGHAuLpRXIM_Hi(4I2EKN}P822G1kBcMi#pd0`}bbuT~GOU~OmZPDY&j-8+x4j8zBDxkThrejXz&3GAd=# z#RGVYn4c>8voo8x=tqY>om154=5m!hr>*;hY{#SDDkkSBWZ&B_3kiJvv|_5+S<4pI zeJr$BP58lyvb(8V+NaCk&}|^dcg&W)hM7}ofE3YwgUCkZ*feZVsQN)jako#Qnu)Qt zK{74*c-@Y**-sWlEFia}Rb@-j?z_iBE)O#MG4a$BY?;EDeM!d4`KcA6cc1o=ye~p;M>OTKlUeI-q4CF}95+Jg3m30`CZc3)I z?|Ry84K@60-<&+x=+b%R)#Uw_g}a`^X@6&AT{HPrAo`t&%nR&{D|vg7ax76u@1Z?E z@gE;&K!vzfPuuGM%*gJ$r16(%V)_dK!W2~zpB6FBQ_%21$|>Z_by_<^`pL$Bs|}Jz&RD>JjF0JBnz7B8oG=D0FrM#xwC)LZh+|Y6XiPV5 z*JkLv+rF7o^w)!+&TJmfNum7X2Q_6YT6XdzDpbrW@r~=FSewPw+pDA9+vr&3dZKg^ zkS`j-=r$<12r5p&9+HL=Ol+hed_=?~$nW+?EX&=$P@2dXt$>I)eP|K?D7at2ACN6D?Mp^f@09|-CrB~>?b0l5AM&$_Y-ThR z#Qih{$TZz_CSM;eR13U6}w&I5;xa1rvvS!(yH6Y7FJ zxjajkH23;%qTg!qD|+l?b;90OqKr7bC=cib#ZWbvGWT&J@Q+zy^2y!zk~A5jo53JJ zk`2K5TE&|Mgq7$yVk}tZT)ObiJom;fEjs3%Wb_~ss^&xkr5u`L*PyCKF53vbHXoF- zS=Dy0mnmkEu%Tl?H5NI@A#Ye3Yu`jeJJPr~v9C_s21p31p`AGOQ4j&|@~29fQU(Xsa%05X(W1m?S-o%Vc%dW1RE#@Z&m-i8jBoP+j+PetqyP zau4MY5eBfnwWWb#X#nm>byX=xYlPmS$+7Xy0o|vb?BDD>rN_@qhn9V=I5|S*wFAQj ze#@lf&u94E8x*7Lc}hL_WxhKtLx-*hHsp)x7vQS%XSTxetkd=)s;@Mw`Etw=FI!8F z*c-crk6n#^q%j?)nA7UbZVZ{KnbPe4Dt-!@q32aGd>NK>;kxXM*5eECa* z-V-IMC1@fEslWre01d0He(*F`N~hBMMdV?s3aQM$0HTyN#YDJ^J@(iI4R_<^LgZLHAgKY z1%mH=`Nf~z@UY>t0S17kW1dx?QWntL>B*7TOs<{rLJ9==7f76$GsaDDD0#e^a;*|}Jte7isJtn`3q&vAOoYj?+Oh|YhfxSohu0K@LS&mo8$VkabW?K#` zD^||u^7X8{Co|t3qHIF^j`!USd66n#GD5!-_`OD%WmXn~UQsx0^drawPi8&(!^8Rb znV=*TM&d+a^i5TtTodtvm=uIw-pl+a@ldL0eVuAZPJ{>=&KzK@xwcC`&KDDSWx13@ zBRryt{O{K&-KCRgw~+D6kly-Y>4!wpz+UDAtozx-v^w%#k@+Bw(ZnvoBU_Bna+F-a+I27 znTpSHV~3JE>WviVZ$roL@< zuJ38_zcaG7*?s^2XJp@aZuIc#OT6{_C_8L;X4fW4d;Gdm+@*roLTlgiSx=pah_YCv zKP-uPetlm*+6c`%Q~2UrWmnP6l>Nid0(nAR;gC;iHV+!Lolo*ZQL# z&~SU>s?Cpnlh;ZG+-J8)f_X8O@!xa#PR>QI^C};B$k)ZEy^rwbe+8ZToc-u`;FTLc zClp?PzY9)xt4?@YiBIYpcRr$TMezP9=Xj*pUvtVl>sr)!XOdFRjgwB(2_osUbeXwJ z9}l?_Dt6h$;}e&Z~G`L4%Oq7=NToSON=% z3dW}}`nyL>4(f3A!wGMq0Ht6Yj}%d^huE7-5VuoK^2g)N<7BsqH@?H4cE?$T*yC-m*RZly4J5HDot*a2L zqNq{egsTL*=8}_Oxv|aY#29K?@3>0EA%1{Flpse}6$?5vS=zoJl{jZeAM#a_q&=vq zl_klY{$wp&iV2XQU5y8miO+lZTY54(4&$nE{8hPxm_|bQe#Vdt!7Mkul$?dr%e0z{ zd%8`iSIm^bk;fsG53daPIEUo=aArdNMc*U%k+wV2_P0)MAg@3L0!^$wo_vRuBfNt zdkI7ey~iDJCfA&jO4rq9F3oKdPCqMry(Pdec`N#3Q+gld@@y1f9GJmD$loLr!MF^g z9dbS-J3=8*?pDH=#RQse8l&JLF(9R1D5u65=C4=Oyp?sAnn;J_@%9u96)JE z#RrV3L<&_PP(=>D#hSta^`S;lLh+9L=ubF){~!F06R_+b(eH$lzfnuy1@L=B7rvzj zl!UD2U!P`<6{l|R+_-&fZ_=wy>drygRRSqDOYQK`xtvwK=qG1z#9lmpp}5onr~$=|u_chtsEN4V=n4}E za4Aq=NekPF|9p$UKEZzuUKmgs^GzsDUnC)H;l^d9DrN&>lVXO=1#L(go<{GjmR_u$ z)0O#NS{?ugYV_4aIs@{elk%H#pX0w&g7cm-6#Fcew+bOnlnR*hva8O|SckO20HwahU_u z>b%9;2qEC+a*=dT41Z8IH$3~pR;v4G8MQ1+LccyN>YY_%M)-+r}x^Pq&>>)dK8l9{+%_SmxJf?@}!Mxe;p9kb?+ zA-!sjTlf{_rb9>)ZTW2Zc;Qt>`+`MaDm9nCuX0@Fa!rXr z9$B>O18?h&bFx8cd#)DVdY^P(uj|?SBmz9))nOv=N|erF-%fhAWLXc(PQ}Ly{V5h@ z@@!domZcA_5FDHHeo^~|f>M)`xiE4e^;dIdhj2|89Xw#9-(kmIdi++3N>fGD zPT`yMX2{vGbCgsMi>x_FWo9sJLTUJMNfRQv`7OjD^Qhq_wG2)kzW02>uY6b?HGuQ4 z_6g{5Zcocm>EsKjT1#!M?}c*RDj`8eJa$JeyWU>uulr+3U5BKz;EKzGN6THw1MXF& zzaCkM;ucyPJ1>lj+{$@Cew6Gw9L661zPV$3JhuMlRMpD87z?4h854RHRgo(dPZzU4 zTEz9VbY=$=di4iKZ}pt|HSYPiNV2q0QMn_q29JE#xp{COv6I#JzBA%Xb)rirYjDwq zQJ+RM>0YZqS6pnm-lICbMp*+w8+>YGC^y-8s%z`E6J4*X6#j(&NqbXJX^K#h*1MR8 zhi#MKHcpL6#qxwutDBkqwR^XQR^6|2QExwZIYEFFNnFj}7?>s)4EyPk)wwbctj2dL zX1G-Q;FWo&d#Q7c{oTK+SSOxIAM4ATO3+Qe_1b;f?8}Q(wrnG=xoX3?7ygaqx1P@` z_R5OXYql*`Z(gN` z*KPc+%WbpWZusq5p*UQgWyZe$_3n$Y?uyY>?a2gsWdb`>koO{fp}Y8g+0t4oxmf9O z#^Is|^L)16<6#Znh^5lQ?D_+H-J_?jPA)K&Vk=J;u$()k3pXd3I zmzPTkqb3+yy5;qHEB@oUfz%5l@OFHN3p-{zSv4LNeQWFK+3=*Q?Hq|6?TcGll&$J- z+j&(xrCB?7CU(mH>?Bk@{rDl}80^1%A}A(Drc6-i{|IRRn`Hhgpi$TXhDd%3uiD}C z|9AW6e+y`#8;P6M2t9*Dc5%ylO>;Lb@|7=6{!KF5-n1-CU4K(`>;D$ejHSS-!UNu2 zw_kQ+)+hgM|MY-s8zpw%ee(zppsCZ2Ew6Bln$efjr@$T4sT%r~9daAec z#Yok^?Vmf(qymqC;zH*rEloa;tE~IGKJ*K?fKsmacYk^_n8?AiOhJ^dQu9?VO%FtS zu_vT+_%glf{rZ0KpZ3rG@1JPH>2%3+4+nmH-hS3{=|2M6jdPD4{{D6N`Td)jM~@f@ zCRRln9wZn*BS7SvXhcq(ZCWDKOmQ`d-zi`HAO6PdzFMpQCuU-7LhB%y574R&OdzD##A*kdJm)bHW+=z=$+B)V3Z&X z(TV6$qqiW@iIO0CNk)mD=prKdiV{MK2qKAOw*7n7v)2A&@3(NTyL|5JI*#-F(8rNO zOBt1)EAgpbz*;9{R{$exwL&m5>=CTRm^YWYW?$i|K)8rOomH6p@AVNDJD8lVwxo$A zhoL(Hxrg?wQC!Xg!AFu|xPu-zz;fVMgC72?0e}fdw6w{&(-WEqyZsDl8La5HLJ6Ql zkKfnMYe)vLW#`wCv{t&;AL%a$(R`{hk*#*bmZj=YhFOo2*M$ouHw}UG>OGDY*^2To z3INTPY;%E)^o_kD6JoG#Gpb!yvI?K--~ve&&a{B&5UWMX(-{bdEFm-2LKyABS=~Oi zFyevUQ`Pa7-W!z|Fq2*DOn%oyb8^ysRm@5PtVlx*M*GS&<57FGdvcp2ZScrMT}QsP z07J87BLzqX^~b9~UVFf|$A>i;@B&#VnLL5`5dVVNPw4X6vv%eC}P*mFwOqK5NMivVP^S&-q2k8@*W!=DM&=*UlZ+!L5YJ8X4N0zdZMB8CM@?K^@ zs@t&au$0m?JgD3f;%LGT^nZcry;vrPl3v`W85;rp#{zoLlM~w66RE3U_KuRv+5Xa; zV?%xyh1I)LX`k&V>keZZXPcj3sf$^mWR0u{26KA_{P}&f)G&NLYdhA4>gd!C)w40B z8F{pvn~{Sp;Zc`~)s|&54*jDepyB+U3=7r^F7U}6(YdMLAG2s4o##3Zkr}b90z6E7 z{kom4_nFkV)sFc6Hw{}t^ZbNh&zLu-3m@yd9O{zjV&8=Ra~D^L?ukjXhyNS-ob8BaYa-eycG?8-y{e_mv?qd*hT%EZS| z2kO=IkZaQd#q6FCNf6z z+LtjqDJzLlFAc(a*Afpnv z%w$1In;#gY6n2znEx?kn>hzYy%xbE>iI%+T3P&QABVet5K`hhp)mRejbcvGHmy#iX zG{%yVq?cocsRXrcX@$!gph~5Sss^YO-jKli9qu;F?3kXZ_`&jY7{e%&w@BrSWQ{Ac zItE5Xzw)N@1^p6zBoS}{t{Gz@QeF!njty>1fl+-U70eff(&P~;@m3R4R1GWrRA?d- z%+FM^sie4pLYkLGC)OF2#F}W=GR)h&lX5S>+j=AO~D z^*2+DLG><7Q`C&cL(ybRv)-*%kAqT@3F@kL@oTxcnB-D-FPF(s+ODdoh)o ztT@*S6&&4}22BRq93WXe+&lmhR8?Lswl&MEr<;bTuVybJ>jEKhMml-zUs7RnvDw@iatP znVwC&itJP`xhg4RRf|Iv8@!ZhQ8w1u4nVH5?)&+p5;+sYUaEV(RF({>xV{5mn z-dEk!(#uu*T$J#}Ev~_S{&g>p8fU{t7T>_^^UjFn=<<_^q}yN4hfH4ZM?1y2Zi?fd zGjTt0hLV|~e0OY7!-90T+=UN*Kb{5gF=1LeAIHZ&sp3M^MIjhn&Pra3dD6tYKYQ>e zNRvtTb3u10y&P}dUs;zjIy_sAjH3;thO2Yx@gpjDT;98x%WFm10R;1U=>2(%Z`#AA z`r3B2e<_+g_^)Dm2vZH1zi%|3vE6=xa=jNnZIM_~YZ{w;S@?NZ0h+nU%kV#8lO=o!4?HQb8ilV=Edd z^7muO?PiRNetIloI2c57A?^@?19SjpyTrAuDFd(SIh9nul*m3E$2{oeWNFv6=abBp zhxG|WGcOqWCV{3{#o2nKWkN#jZQXcwOae)W#AHacHl=nv4g?l(rhpX53H>8`2tzOQ zui0vCog$cQdHz_7^Q5xUphMzWEPFt@g4P}~@)0u3s_J(|$RxHdL7#XC`p)Z!gh}q{ ze06}Osoyq|9B-m$FQ675FlYdNvFJjD0bV_jcA<)(52mO-V^4hza=SdJNt$2_PpRRw z?sI6~$LFyN$Wp7z%(UJ?X9Z2`yKBMRQ07`baFq?%N7=p6k!N1O`8jo!yAQf%3 zavEb-A$VW@B&1?CdHXgKKQK@frZ|OLJ;fJM30?&*6pq?0;wV=zA&bi@W!4D_g{6 zW~$3ByRQ&Xpr>0 zUlaoF!wO4-rJ?JULmpMC5wTRkpy@(oFY*>Rg{qpKQ^%LaXyLot3#_`uq+&}r3ZUx& zkh3iWI~RBY@o2)sf|wlLhxh!j|DCEZ6Vsn~*q%8RU8%K z{HxGjTiL<3K=1jdh>nA*F%|9?=`dh7KwTF$ldd<%QE5j3Pe#C&eqEbpM@NQM1{GG` z9gD28zW4qS$kEmwSSu3!-DKfH@1RVh*ajS)Z%*~1=+Lq3sjjL7jkXNd;^Ey0lZA&{ zHJ%4PGTDZxd}}3Ztjxd8DhRTub?1JrJ0A?5f zmrF^rP!Q{-z#8n}JO-3&cd;zbKRuB*-#}z8-?aZzXx*GzGzyZ?6;27!f?ORy2Q*)d z0H`wbV!Qnaj^NR4jT;#PzJ#=ibODQEdKFnn<38U-I;Va;mM7{Q5CP6Fwd78T=UVe zy~ys=@uF*~LJ?0rUy$8}!>56gr#ia zl_DGp7XJ1sUCjfO+>f&5<~}IE#RJr%9Xb_3^6}k&mMDK?DJ>Vem>0Tzv6S1>j*#TB z{!UbdpzWTv95ZJj24^9Ow3XkL(O`!=bxA>fs(K@(6a+PC{Nk_g6`SB2rFw99k4f?| z)oLW{WHl<898(t{m*<;|d9bfb1y}Fi52ty<Jde}f*RC0R z*ru0U5JHuwGZYHuQ6sS_?b?>_2r*HsPdHFDYs16A`apPu{D}!^12Ge0Haug&B2h5+ zTbDs1bM>h8M7RoMd9TOb7&wImD%O4Al6SOvRxisq=ieYwRyP5l*H%%j3_FXcF9mLQAEW~X2C4HxE!-?Tl4~uKY9776KpvSn z%lA3Riv4PBj~2eqeEdqiP*C>-Rw@RHSn^Ro9i>=zTFjV&!=iH zXCV-ts3|jM-?4(K1R!5|$W-{j_(eU9Zz&K~gi=}5B?VP=T;u_@DvDO7XVf3>4g#2; z0y?aNq#8iLA4oM;!*Knq=0+6}7~oVpe0NBvT0$V)6E$fg`YK;ob(r&j{99C3UIr-l zpp?0PGwc9hBcIjn-6vogG%$WP)~8eD&2jo^T;ZWIr=?asDA>l0CF{*;;w^B6@;O$|lv~UVdnZ9b z;!CCSHb?sGNvb5pBTP)@D3p_RswGTRa$O;89EX3H2+9ZHUo4mFa3lYvwh$O^NPD5a zY+I=NY(I*ck)HwPzjSfLo4$OFu>qHISW9?qskhitss4CjS*YqoUQkSDQJt59uOnn; zYpVaNS5@4rc^aKE(x&HcFY0`3lBH-xw&KVf9cTPttIl^@?LRc{MWya01}i&cfNWb5 zak~o^E=xr0Ug~zu6)a7DqO*Ax#ag8RQvB=cslKnNW$XrlgPcq1q#>SYg-V2zF zVeUR5R6SmC)c!5%_5cT8r{{@cHC;=V-6tr6RM*MqX+LOB;6uq%#xBT;%dXBzwS>P~ z%O^KIz4-AD)gHJX?C4~rlg4l}7{+5TtQC1O#x#1yQ!qgR#DzTT!l7>jBs?Bd&X5Dl0farqka zVJwveNE2$0;n&WhzLMAvU^SQwRyD`ngIs={>}yPqokhR%PiN9{=l~t~-AD3)?n;DT zmLI5dG|ivYkx0H16+HY8`m}d+#_lkR2)CguJd=_*6}OY4^SwqBgr3TevnvGHr(n$a zDt^g+HcPQm*l@N-3QgUErV#mW-161A%LDtJ0#(>?Pcr8yglO^!Oj4IQ>J# z9?v!OeXcH3?}Uq70PFIo#D>LbCAR{y4m)TwP5gzW_-R5^vsY?Oj97{GUmi;I8MMd@ zL1z{kd$;@^%MTbaFB`n=srf7{h#fMWNdbl(=#%%LpmzOFyjFPOSD~tp_|8Ic(>;H- zIag%H(b2G8xu+TCn(~u}a1CpmO_+_XQ`nh}z2}*krMbh+mBCBmmNt~TlrJf@>H#-g zt*dF=Q1+uHEpBC{5OFvr7BQ|kVb!gDlkd^ttuq6wlB+k!pQ*vjHAe+&tA$6JNl)6X z%7zs;x%~y~HYp9iRJ~;KMO&mjBbxXN&Pa9z3uOg_bP9hnXbU#$1+sK}&?dKau3f|a z4VbI^eI}lL1m;cKsug?LrGZmVFIV)9!a$7;uL-V*%it-2 zlGqbrk>1OYIm>X=j{@A>EB9{Y*N<_pZaKegTf&{Ghpjn9{0&?8gaq9j@33_a-@Ns$ zA8_IK`nK8OJ33#!y}onj-=Cl4I~3k#ag>7iDX={nC~T2H zOTYtoF3?ati$tb6JV+`ML+5Rg#I=hjLo_ZhuvCj=0YP_a(@1UR28$F4f;+9tg*N+y zMXExbJN?Z_EcZu?yK1}c*KS{6`M{QGSV0eXcBGD=uw}Y2!Go#tLPu23G6PrV!O|9~ zE9q^S>9p&?HhiHgn`)WmDd@@ZGExuOV43Yt@Z@@Tp@*8Vymzb4ljlpMzWPVYoS0qD z>whow(O|3GBtZf{eUyQYuvK0rfgr&1%fLX-D!-tPAS4xKcvaI`P`*nL(fDO(k!n>~ zE9fO=8f9eNU{%yY@RD%(Wn@2LRs6Kh>k9WXcK&EpGPvs{bNiPuBF?;M)R{w0%-4iB zgQ0W@+wkLY$+Mrj?PdFACGs3(WmE4HdW}y@*K?eInk-8x*oM}6x3)%g(t)O8KC192 z;Xr0zD;Vw{2$noW_{@JA!RhQ@GAMs*oS$8tSG{Cz%z(~l{I{s%Rh!Pp}t9K zpm~iny~$4xBNq1uOb?z1+nQpiUk$YtTI^;Oq$KL`6z4ZI(eH`c9jk~0jjmJ~hcuh< zhYy$kWA;G4Z0d@f(J0?2sP*`3w@xv$yol;jWs;Jx-q&wTh(4{`TNH7cKYZs5&p~H1%lpP0w(-cM#qu5}#c?;AHo@Crs^r_Dw(K87E@6awx2= z+*S8&<%pSGWZLke)BQhgUd^mnV^90IUvJ)jHan|-ulrh|ynL&P0`)|rTRQv(?Yd>6 zvW{(i+YL2}!mdg_6HYy1x|TXFUaFgquLzg$p~V)cUpSD2%Sh^0D|<8Wv%}L%)^^0n zDQTlC==aOVS+joIVgJa=qk=ub-uTP63vwR^$~im*BhJUmZrrTUGj^%%G!=j1 za_p>POrUsahf6n=jsGLQl}keQBmeRiKdo>$*IXX63;Piy_g1-=k+rw(MP1G^mwUbU z+yj<_5Z}K0Yhr+?r-#GYg||&UaMhbW82Q_xG;+B7nnrmD_4QM=O_#l4OZTF(qI`~} z$nK-KdrPqlClB>CcDaI@AA4wg_sHY=^cK0XQdlh2+`Y5u?$bpYb`AH6;{JG+|9i&g zUVsPvQ5~yD@BDku9|lx5$USYR$^W`#L(BdWe$+(1jboL581&isA>;3Snm|QMks7m_ z@Lyv&n*PJ};;bP%ef(sR_Ru>LgBu^D*rrahBL*rfnX?%)-#-EyIoX>AWX;+PlLK=| zLQ=0^{sm*vXMt@B4@BL|f367Y`;LF``|MD;RxdsZ3+g|5dtWrR!uDQx(cfkzT^W_l zqc~a39rfqlOdAdXf!$4Z8yiv*-R~6FddlDEy*2!~zWJmARh2V@kGQiLH2JRdl}OFU zoSf7x_IE!rhW_OjdX~TatJvqQvuw>NcYj3z!shZw{N$J4?;(1I#?asaFH#T^%~PP# z(~!8{uRN^STU^=zIE&!99zI^Y`s$5C#ai)s^oNHx6~9MHB6@E(@)D^BD~jwRe8ZUHj#p~Ko(N(xwd%=j zQg;d6_w0No`({4%$b^jr#q}-xdWAcjRrnrxuCzQGaQi4I7K!y2wfnp?e^Kiy8GU!y zVC(Ju0MCAl8^f(wB>2{F>r)N8w~ssu`oi=3O>o8aIN2e&zV`OgyZM8J+NaUl3jWwO zsDjj+ot}V2>K1?Yag6f!$WHp9CmX*t&efh(`**&@4t_k5b#kh2|NZzh`guw7vxl>i zBV&Vcf6@k>IiGJ+icTNaOnkB#=}vOgdJ&D zk^Jabe(i@AdA5gqiW89rlE7jl&rhwM?g{U1wTwO`y?r`_R?({eB)>e+{|-L`-6~ zxeOOvWc;f@O4xX2J48P;tyO6}mJrhmITbbIdTcuQK!q8Z!%nFdpd-VqB5^A3PNnrX zb3$^FnO;ZQ`LY}#dCgT_2Xe};n>_lvkL%i>Zg$$||NYjU=NvVjdm6elkg=^}9y@hC zM#V(}o!%@<#(ft2P5tc5Un>zvJt&}R=sfj0i57mDMn|Ff#Hp*YeeCVV3*V4Y*-yIu zh3Kww!#>98n4=dTGBv6cBvAzYN#nNZOf3)b5nbcP-?Wc6ZVmMVR_aLZCeS6rl7vPY_nhDl9goG{_@piiCr#Z)suEnrT}i`l{IPl5n9>;NPq8s? z?x)XpNB26_-~yTzr^O^E=KHO4yZ|HjO=T$^BfVD9L4L`GjjQXAylNq+in;!dd6uPv zvx*Bt{@&bA#)+@PvSH<-$^g?V$zUy0ZqdWcr%=z!v$e;#NJJ%nx@Ck~;n7fQT7~nA z2Nf@M>!iO2h&>6BRFmMHwJ`gRFp*Ll**sFserpy*E40&WAZjfoJ3BjMegD~9o^!1D zCv`4ro%?77%@6#XuRaff6d8wiXCKe8pGJ?iRLj^_TlfYneB*u!iRBzRE_=zXOkSGj zq`WsEX8?VtnX1qbL9iI;;OwvF@^3M>!w3iaOA>EbjCEKH9yX~m$>*z#8~~PtJR1|W zIh5-}Yuq^`G7R%SEauc$(i=5{^Q|^JmIPi}Nxid@{$eHb*GiTiC&z=6m%=G%;E<*` zjG_xp=_XG3HV&1IyIMx5w&B!hvTF2B{*QQeU zrnUawb;;Y-hBqI?{~w*Q7z)cPs^sqfjVc+t3!@JaWBQ+T)+;}Gs-yP*tz`-8BbENA zQ?_ybA^QKIO6r=r+*2B^sFF&f?*CmxbERcvbWx==(u=`9^X1MNc2F%g2tj-J0Vy|4 zGb&qb`2bx({fF;n|9`5aBEPjXjwGm=dx~<@=l*rg){P)fU)GUsuCAasN8l z?f=wyAcE7N{rkfSLS2)xsU2il<)TYijCaue@;zbG>3-Pp3d=i7z7JZ&c3Ec^e5m(- z8InB*+nKTxQ%qb!yOTgi=8EG~CW7UbPCQ2%tDqr*rfZVJpF;-&>2>(X0;=XmZlaIm z3@j3X-THf42CiNDp;u&Yu^s9dlk6CqCcKS18ZK(cB2I|yUv02!q|fB)15v5B!hpE&qw-!dY(8z}p#a zJ5bWb7V-lbHomlvQD4YKSU`A8^pE;>i~Ya$A9(2Op!4d(65|M7r18h1ZT*|tTdcD$ z>a5X5u&3o_9Id^qrcE&RnVzuM&V;-d6b!}m)|A0+_5R;qu=}(I`^$+AE^;_R>*5p1 zQ?PGC5RTuAb$Cq7LJ-}R9M0==)o+`&#F2cx7-+XEpA6TkFHK`^M<*8nK)p9WpIG|8 zFI(tKovo-?5@ihBTq;Kx0%SXWXq~dwh3KJ%;y1L+IT$N2#d(S{+!+{=J#^&^vwkGL ze?%LiI!=b`U+G~cl7Qfm4-?etxVDS!Eak6}+SJ88mY_tV$Dh-bNhTxVw)m?RgV*3Syez0gAp3X|4++Oo*H3@ zhI!626=z2N9H~iePd3v2aAANdfIY~Sw~6A{BS4D|rpHqqr%Ac?Q!w7ROala6rXJZB z)m|j{>aNnrhug!W0dpjc+q=$sG}zhNkXPyDb+0wW&O=LqnRav)R2vH2ZY zkPkfesZ6@5@kjo5JZdrGj~WmO{0uw?HQGo+{*TTlMs8b>j zx-RM!%qM=gdMynGudIeoBvbP=5>pkNMMwXr-)|LHMag>mZv>gGLc`dd!^3CbiN=>OqU1H(^KE;*-s60H7gd`T6=}$ zoXx=0N$pA+Qunc5G7`wuURS&rN`g5a*!FHChUSE*i}YLl zt@MAU5ySJPs&ri?Ej{f9!2hgKT04-ufl?ytZ*j+dFdlMEKA-PRDRi*JeX-4i zc3Mjk6qW#$V)+q;H{79?#sDm7kcxC$09Br+dcyBMhL$nWCRGSeE3rO$3Q!s0HdFMR)KCw=Z66UWXaE?#cTfh&djeh2o z@?Zal$notVP#`X3J3kgy!`Uh&0cpq3lV|mSM2c_P>__hxce{t(UV7QPtx`%$8da$3 zNCTPJU%&If0`p+{1q#<_gSHJX~z}dQt6>4SCen8;5&fsG~|Fa&su7w{n;arw#i`Enj z+3aCC+M+(5Zh;&OshGyp^5FP*+L@?xs+HNdSszX7+OgYiR!-lja6(^c4*SWbT> z&gab2<6LWY-_0eMLZhPo@^r zBsEz?Gt!3TTWQ&m-IKeZ?8sz(rGz?ODMM%?xi47=IFgu}bOa^Z8-^EeV4Bl%E= zZm%fBKVx7Xl66qE_`C9?ONRC5mSnIQDdo=C2-u?UI2)gl7QT?ibvvSi zj`8@N3_qT86XZ<#3B7zBi5Z9Fz;p~1K}+=^WV~@eIJlfI#F+$SXGKLO5^9g3i59Nr z@}OEgvO0n~4Ify7_jmT7&g<~h7J?QpL_j+A>~aamuibTf1tDpGL_YkZB^}doVEI~L zOAw=pi2S>N52131OoKFYuc00o2&<^}A6jZW6ol4}SSfd^d5!cxQSFTao?T2T> z%T()GY&X5JJNp9N|{kA@vOrQo}u?vD8)hBq7Uj~E4)5J4iO3?)003=kw$ z7NqHQI)IpoNW)=2v9VYG(wr9^#b|_0W-l<6ZwFf zs5G4)f|6BkXFWDLwcO6KX_{wTOgk*We9(M7ussnZj6jSR@iz8B)Udb*27tFfV;7el!Q=M8HKi>aYaKn0;}Bf0V}3Cqk(j~ zNNtPcDZV=zCG7iL8*H zsC6pdGJAr8^+d>WFDOkwox3MI0u7YZMUF;5cG++Fiz2kEGE86w$!rn!wv=h@5NRPq zaJ(6vHj1^0LIzfW%S}g5Q}uM^`0s;@ri!yZDPAd}D41G(Uye_^AerP2CgUi}DIliS zmT4r=+sH5+8YQ(x4c7&X_oZCa-5q_SZ`H&r4MOQHQ0uf)&sX2eZnwXHrU&-hi1zsU ziliSb>9nD&0137p1EZSHY@_H@IRZ&(Wu|{R!7VE_|-AK!C#r_a)vj#j~#mbAv z0^eB}l~bI=uuO>-0gg>mm80SQ`93k7j>uyu7$P0F7FTlYQS-r1atrLksNmly;<`vZ zeagi*H~YxraK@0@)45=itTD&nlU# zD7I|^FPJ@I+LcQ}AEAiU_ArH+HzjX(khhUS4ZY&#odCkwHJe;?KSb$G`pN#m~ANKM9R|8>9V3Hi>Jbi)b6a zzR^GevA=Q09S8D;uo;9G-%aI29cQ>ir2W!<5=M5Q^ot}0A6s7t0->FP&iF*RS~uH# znVd6@p(&w~s)qqoC>)wbl24CZX5V2T`Ud;z ztxD#)3fm4c(fW+tY^<8Qc;P`!0=OLCneTP)<0Dneeg4_{>*AS!#1c!%1!{wndME#G zI&DBPg|S?H*OR9X!#`OZDVjEl8Wvb@rpdz2(jdJFmSzGE_Jt95nt9+MX3aGe=en^3 zR+ONAn6FNP| zxI(%Rs)%Ebl;78~dd1yC;@gXJn77Pk>%7voz;aARe^#9P)}XGIeCFMwXJOX2iwn?E z)O#PLM(u=3H%cFeQ=N6UMihItjPf~>oRa~C|HLJW(O_OHud+$mA0Tgs-K+_Ag>@0u zPMZg6=Z&M&O-CK3y$R-Cg^puZp(gfE?lz=ugh7$d-i zaYpb1Yh`H)^dlei${yX9*$`s&vp4y=uY`XMC5NXe{j8^xr4mBr2xfKLg9rID zxZ}Fhx+WQQ87k~L0%1GxMWus!D@N%+>9GcEQSQ)EL!>$yDl7oZHPJ6gRSD8Ic zNKhV{*tb8g36Y={WICRn*I|uDl;F+chr0USG$EhtI1<|l+U?XKoC@!K8}3L*f9)46 zUr>@8qbOrTk%_ZgWzTeW%&sIT)|%0+Y0#W5UvO^2>z_e9#pZ6{`PJ2^IJP8zs4nwD z;CMi$ERl;Fc&U8=YXrvLKM_5hIEm;JA5e7mmLH+0lJed`k zFUp2Q5o{C1cJ!>0RiH>bb$NW4YHCbSj|&Y&$y@$dU3+jq*+`o~WX=M$`kzO7s(Ouk zW``eSuX9b2NziEV)pY2qXAwEi18VHbyCQqj%!JY%dUi*D}RJcmG`ca$BfMS25Esw+zr zbg`OZtRke?8=)ZOn+^Wm%TVvC2tA}1x^bMv117<`6fJBsI%7l+u2^$Z^QMOXaL>Vu zm6G>kJn{&`qZE?yX|%QB?k<$U@mDGKS zCGMew#P&ntsvO_iWW+hmNJ$(91Ty}RO^ol?yqh-T$L+cU^-wAXg{m`cUWlRB*9pTuju>WfA4(u(W8RzAqQ_2^?-1^JUc$MM#ca+@5nJ{LUY8r$Bkap#SNeqKD0dhd)i)?Wd9 z@T5eXBC25D7qe5@HUBl8@ZX@g3j3#{?(f>%@^Th;KD>e6yKQ*I@=S6FAFPDE-Ls*o zqQH~RnwrxJIxFQ4`PE5K;jz6<#un$sr3=dGG#r&QM7rU1^})N3W#u|Wu|m?(XXd@qNc++63rbO4^{d561~6ssy9_F6;e)&nL~x^ zB6c>;ynvj62?B3RjHHNMD8M`D$GxMVMV+{3Ys$5csJ=G_aFm5#lOF$LKTQ7m*;8tV z`(KAOr~c=^m2z0%*=ug&{2MlGU*&3*cg`O>wt8;ArCzKiRMSAlPTeKwLeeN;BJ7v` zp?hv18WG+PpB9eYAT%tTaysmJ+GWO)qL+UrtM~zk9Ar2ioj`>^!N_HV{`|d(5;?b! zx;Oa^v&F(HhnxN1KbluzQW%6BR}L)e42z__Hda32nk?(h>K$1=F=Kd`t^1}aAlDIq z1Wu2Oc0hrenGy8OSS`SbgIxOg({ZkUYPsy~aQv-w&y)8(q>;w3);9gaXPhDM@?Wi72R{fDN!KCzYt0J;_EDt{B|H4dFW(?`uCWjf zfGG?tda}WNRtagwM0^PN@)~XMFqgUWlGTS$oqNu_d6${T|M6mlCE9%nG?0{p^3A5t zd{b7(G{}a~U4uHy^?{&oCFAKkiyUdKxVFdH^5us06aQ4)nUa`25-C}Eg8>>_$??)P z?@I{qq-{pdXvCu= zH6^$~!X54V9ZfG5H0D5sHcnuYx1^|5TA-2l}+@-KV79k~2qhHdvt>8GL%Li`>{^8(5o zd6WtwjN3rOR1H4@+_#mam>PCS&@Kk{KdtL=({#^3OZHwTbexd3jyAMX!bkG%;J+ zwWU??Bsi2`gfu2Vv|>X9Y}<>-CX@O7UI0-;X6j^o0v{{^z}XaJdVd4Jpt^-rk8MKd zEL^#CMC`7v&h1+c#jrK+dh0?@iDWi$7DJU#Bz1Za{!lZ^&MVA2Rq4wav6vOfg zO|tJ%K;YT!b1b%EX}#{b)FybKgpF^$)M_rZ@(`7O!||h3pgv`q{k2~(?*dQZg;Kl# zJRNmz;;V{%2xxq;iEiD<3ROsv^8DzJFVjr4m*t+ zcrp8nhJSzx1*IDGhRAVY$*X8F_iNV(7n4HA3zZVi9uMqWR<(K(hwtD1BE+$<`Yaa+ z?n1o? zn=#(LUEwy(DBnmex?st-GCTerGKZ9y$cq84q)`~^DiRoz#u_?_6Nm@caRF&eu`Xk) z(!ig)e54y9Wa8HZdjB`rwhAJIgPQ-Ed=;9SeFV%8^SG9rJC|`a98!}_gh!xU;{IXk zQZ1|_NbX5|+%aM_y6Hmn?g>J`#r%PVEk?zVO7|-p#*p4NFgzN_wy*}{WcP5IHY~s& z|L_+)m#PtV;DbJvklB>6fI$V}$LMC#?eEc&q#j=$87ZB_fh zw+`nr)CXLYv+AgE_)g4jcjyy$-cZ{ zmG}txs#I5Rr0iB{To_w5@TowiV>ZED4uIhpZ=b3 z<+5R|Ctv^FmEC5ogj`#oXQkWsT~*RBz7#xa|1fueDlG6`F;7pyWY%(Z*++A=>xdh- zSB0iVveFaWP1U&D@vH7E<~27{VY$^1x7(Vb<>gP0=Gh_I_;;Zer`!3L>@$@iD{Qn~ zhL-na6X}FRN2rZ*6Bw$v6f;u0u&U@Jy|OMVgo%YT*p3>5Uz)_U@3tx6<_VsdQb>GGqK36iP>td9YGH;&eNyxTGEcJ)q zHJqZPgyEJTj=MBibQWPEk`-q#r6o8eh`F~`x=%}}@&I`?)IeZD;sX5(yRC(Jtq!iOMTTvb5wzk17afIYNNDRN-Feuf8^b9nGpD88T0lB7gJYJ zUF08{A}rUtgN!8MRoDIQQa08f9_}*&@4lO1yMD2o{1uXU&Z#o-PAbClo01U&H96LW znW8a4QrvCrv0)Q*_iRXKTCx&{26M)<6;!DKmLhyS`{UWtV0{1eghvH;ArjNhrjMcq zeiDvtzIRf2+dek8-|~4FRoVLBBa-X(XehJzvpvh%avJ}KnF=?`tEoTY2VO4>$@}%o z&}y7QSt{dm;J*jf%8(72svQdKC=SirhMHgi`p>PV?>uF)%VAr;PX;q`2RW+e13Y5#?wB#>uXgwub zt%c<-TM+7FAE6kEE;R)@G@A~oyq4!xS~FwmSb3BB(bm%$u6~rWBxgV+PhI7t z0D6~JlO{k#v$gDT#~_0!Pfdr^_5w_!i0)JZuW|#sTVCax; zkWNv$W9UY@V`xxRy1P>bX{5UykS;-5LXbxI3P`Ew9DmPwan612bMO285A3}^YpwVz zR>imDk8$rzNoctuWo!s)o5%c>-Ulir&`&4&It*fuq{BtWXo>nRez(P(eh7@pC_?lW z+Ee5E$7McN9~>Vt-5YzeIOu2zi&YwO;nOI4tWmC|5x+L3yg_|ypi!NqAuA%1?LS(x zs1&c&UD>C$CzDs51#?aqYoym~;nQq=Jk?w{aKhWFs-)Se)cdZjGCyi^a$&OG1Lila zX&I_HaH{$KUK2~CRns)qi`4qCH zYvZ;bPX@wuzVhkpKGxaO(%HA1t#r{k2-P`C(m5{DIjPfmx2pARTIXy-=f6{(@Ao?A zM7kIBx|e*qKOXB|Y3W{D>fU(h-iGSlCF%Yw(*0GZ`@2u~ep>g>hVI`}UEIAc9x)2= z59{DZ5y+x|+9;3}iqI1Ueu;u4qoBnoSUn2fk0P2u5pSYM&QK5jpb*4*qzrmw{CecF zdPr?O3M)NIPd%!adeq5!G{t(f^?G#udh|1T44Zn4XL=9+=rIxNGc)M3@awb6>a%I< zvs>wNc`9c}q%s|wK&0zz^y#5S<7D?`g+o^}aDk^ssyv2vzm%cRO6HyNGlr4l)bQq2SxhR43Voswl? z$KS$y?ZZ8ZXI?2K>lt;Mfu<7I#O4fFsN(>qb$OL6dG-_<#eyY|;Rz0#fEpX)lD_ze zmkAVekme}Z)w>26M-yBh(UfrLn2)kV0I>-Tg|P`m!j~yYul*q=`-G45#$mr!WGI%B zF;9|eG)m znx!K%YF14OdnW-1$4Rq*na3X*##5^Bke>MliRE$B9t$T1@y&_d@w~S8ls0 zB^v2;Wmp^+&PHpJa<}H~2#74Me1S~p_B5$*Ssh7ANU#SeC?piJTMo5%eCRiQ>tYhX zFVUgA`oO5WA2f69KA=(>e)!IO=;en9nS`7Ufz^NmB{iXX|MGg!<5pcwf#FP{J`UV2$5U)2R1Y#%f48*`fb) zsq$tCcI)Q{`Rro)YDeoyPp&(V_0Njc1lO(Qewv<_*4OL^K+2Q->@S|1%hapkfUUJ0 z2NiEP5;hcs;ykw{cV{(jOKs1s&v0bw%@u!kmRKZ_}wh@;o*Fz60PXASKo zZEgZM#wXQ#Cymy_%geN*mx|w(Z7cS@|Iq&|`r6u`!iei;!0l|S`&!bENPeySy?z}O zvWj%|Ej~huB(DN2(b6uf4iP-nDVYV0&f7{oj7k}^iZ__UOv4$d_y23(?`qo`Bl>KR@;Q0#abMx$TQ!p+ ztqDF14(HVgS5LKVl$*&I014w>37@j;?|#jw zyqvBz#MYwo&45|x=l&}vgD1l(Yqmm(0M#AqwS=7~MT>-%)bC#IT=D}tmIm|rwn>OH zZ=}|7GRmErGH}U(co`>>pNI&Xg|+|s?P<>seT3sxSR|$*bh6cIGT>J;;yq-(y?!7JvoyHP^JAiM;qE0?N!KO(|!sMsS?AB^axJ(=e8nT}^3Q(eXUs__&{=4DBKiy5cYVk?n&>-a(#KCtmJqIB zyei${2FhPEC5xcm4u7_Yjllwj4gm21RS^UZy-s<^zm_hn-4$;bdk3C*+*o9$SY^cJ zyxKgDw*zo6McB}B@+_zQn3rylwC)&bdlF*NSO2hS&1!|?>Ceq3>D~iBDa+#VlaIyE za3CRP`t9(Rv+|Fl7LG;-g4#d?tNx zvGJsBw?0X$bqmRHn#F$@%I_s{5GUi*^_cneC5PU(uH!<_tdN&zs!2Sb+RyxzP+a?8 zcn9~1)YR_zs9CQQlv46ymwc(eBk6ge1sy4a*}5r>dVcgW;XLSwa#GMg@CN*NTgE-7(ibuYjg}$&qSbR?Y+-iO%k9G5;Sb2`6C`-&=Q! zyRzV<9m(Ne7|p+&F>o|}OA`Mf1<#tj@t!x*^yT#|GHs8S5pCUC*R)u1nI*TnopcrZ zmUHLFqwCgqN>uiyD|B-iWK8=yMdWurqu)FD`N3xauw4qI6yO_qxM6I^ zrt6d0wQKdQn4QFyQZl!5{c6uPP^lokm~2cz@mxBj!P235}zT!mnCTA10p-`GI@+$`)h_pf?44 zeD}R%^*iTEe1%h<4CPMwja*0Nam<&Xk4OGsaT1%#RB&zl;&Oq8uG?S%^!J5qautA> z(D+lFjnj>sHOcpXYagP!ANgUg3fBO-olqHahEH=p4b&B677!IL@*&8| znM76QT7>MkgU5&Vu_Ovy!c1L$ocX zJodL)BjeFwW?ALtrL87$^r?)pmN})1t&&lHKgfqLCsT_Av{bL>)hX1%Xw>$!e(`{c zVY|kX%z}zqF;IJsFBiIinLPVqg%!8#pF?@Ru3XtiMNII7aSHt+Gh6WO$M=t5bsn8= z1Y>hEf!OA}5YU2l`+Fk|TUC9UU}tm1$N4!7ehO#X`cT?r@EetXf5VEMpWMONyh!hT+ej#aAyQJY|>PB)a|ai%^uM!L=C$8WInaslxJTHiB@57~7zK z36doxQAegl;zZ&rYWPpmtApZ+`ZY=dB4BQYJo%H`PfvxKFmh&dQaP5|0H>cHi9`B_ zwtGlB9x)qI--Kok{iqkl6r{O9ES(u)3`Ge9)vF9Y-ON}fo*RLmlC>?_ zSzP~Ix8)!<#5YqMNu$ys=RY<9>v>H8t3kGG$QA2TIl57mr>x`9Y{REE<=x4q+>}uqsACaKdbu&O zi@n8U=BXz#XYK3OtnZ%g zvt1@V%@I6k+2n@Jj|4*5>eF3ml%ulie-d^pDBZQwmEm8ze#r~B@+^6&)=>W)tpb&2 zR#8N9WW7YkqvUrqM9w)<_ z8pl^TP@_Lv<>V_!PaDsWxF(GV(!!_%t6#n5JvJFF`+V`1rJp7YU)hnvZ9(#+^9hJR z!6+~Zh`A*Fv&Gx2wz6q&upW$r)W)vpkPF5k-faNgZi~b|F6WL487~@FCQb=N(M&q2{`&zyV`(^RLu%^o~tpoqpOz4rfalJMcS`?mj28DOB}&fU9Ajom*X2J z{!>B5d>5@qG?AoK>nmM4!f#d%&H5p2lnEybe_gnt`9)YT8pkjauc^LIbO9d!orM*y zv^~(*@T2&w6mCg(61G*%@*z_y8lMP}Suokn1Q%iwd=4vjcL>4Xo90Z)mO<$Z9W+yJ zHN2(l=TZ@Q&N?EP$)jL{Phu@t7Kw=kHv>?{c}5vdLowlBHK?^dt1IN$vAY+^Ohe)@ zs;#S4Y3*Nj}ptESGmxC3i(<5 zp1!t%`Tjxl15#=gg$mr@qU)5|cDIduS5#wB)M2J#X>y04Yu?G`o~Q%+{=JJ_JAKE& z%W~0ii|vfAM(TEfZl6i#Cj|VADIobER*1!oVWEb8RD+ghzDBTO>Ce20=)9e5;&u`8 z(l{;_Z|MrNjlk^eBzDcngq%{37s~UFJnSx>4@{MSIbKejS&a*1mesJ5s5Nbgj72M&Vy|prZ~^GRP2=opT_i?LCB>k$gcwN zG6jT13dQ2(uJ}0%{;;Gfex<2roIJG)4KaVUw;G%XFufy4*15BR8A-V3b-i>< zi^HCAqwKYUb9zky#P!l?@k?^|5q*q=q7qaN`ei0qJW|t2&%gji1Exra;XO7Hl(3px zVm8Hfjtwrf*T@+4);O)BUox5em)_HWbvcDYb><(vHU>)MEgM}P9JUS)Z9RU?AGs? z%iEH)MN;{>X45Rl9&1SsJcXu3>M`a4TdR#(eA_Q1iZnhYty^v7mOiwrZ*>5}&fnD9pZh@!gNz0n^PEU98xv6ddL z7veGo8w4)leficD)mfZEc)eCOk?3g!T&<6*=VtO&vTlq5%iB<>ZDfnl5YukbxywC^ z4Nc6G)Ue1h?jKLcUmO{^idx$vEOkjf;nxT%Cd*V)83yHgU&V-*n!rwq25nJYHn{1H z;G#`F@V6!?V|{$Sa(5WNF31jv@|LWZtt?S#7u1(mC~!Nw_+P>(QoWmN4ZAv;7@!3a~_E!GG74;Ocn8^Yjy!Mm5g4M zTE?y_MT)9bjg$6d8>B_4YR;AXxc`cnABNSX3W%xq%qU@oH*5t-1$<(%g);K6Z_M;& zj>1(nArxbRl*WKn#ckd1XQoO~BMXZU(&|>8CZ%ew0@Lc$TLX4`Oqm`Z;8Ne$t80!Y zsPScy>Ou5uqGW2uQU#|q=m{0@OfRjJdXQ0Gb>l14;Ckv^?P$^DaoSMFsikpagT-)Qfw)zlT_p8 zcA04j2`pzU*eLP!Rj_z}K z-sj0DKMTJzFIl;&eeqB7FkI(&smznF6d$bnwBZ4Bamk?iuJKVWL;H;R6I(6*IIX`$ z1*c&z<|!<%ObF)j+SMR{2qOq#@2<4+veYs@9N7Q${r0wxhDapmi6rx8@R%!{ooz~Bw1Q!)qG~2`gt!5T5 z!!ebgCXU%OP7UJS#+vY~Jf=KG4WwGf3L(U;~wvs0Arn)yqIs8xxOV z8XojFfnO=?o?a6bbOGNWMok6D|3Uyi&dZs1K5iO0X68M^Uh9kYANx^Hv|!w3>8svi zqSeQz!nO{Tgp)yEQ9)9?J%s7GTh1Y3Q!fvA1eS0-&XUXCBwAIXnvdh)6R6^AO>8sBAP(b`Hbb zMX~WjV~0TZg;R}6;gL9MVDC80z%JHRj!Afb&6UewJ4;WAFIPWnc^02|L}6HwEn|vA zikr>e{slTMjelW#BmDSD7$1g^|Ivq}@HLl}%eM7lA z(e%Slac3%g*T&vidY^(W#<{xdusfQ^DfOqpo^O(pXt!pQw7rwikBpU;|7!pfWG$av zqLRWIPkc`mB%i$&`u-gKJ?Y$A8I;f_H~sYJzvqw?z!>eu0L10hj$#^YO+ou(5yQ5v z;fVl-->5EM`~0~}Q`69+2WyY+@#_Bp7s*QT8-r??u=8W-1;_%)pUJrVyGOGTr&$+` zpJvsv{9)crcz_QO;uXj*eEtVZd>IEgE21eu#CHf8Rr9rYGx<}Y)BE!?xv$17oXBTF z*BU-q*=9j1K359qH*U=;#I?!eOl@d%wyjS}Tr+KFb5dWJCif${ULl5+W`-|9j24T( z(3AABUW_;)rXS;QurOh_F3md*X3iFt!_6$d7S^Z2Y^uVHx6N!OE$q)O6{-a}yoEVq zeK=nWbH%oBr3!QBws2DwbCk63^a{~7w(!0e<{N9_`zXx6(!&2mSm3Zl;Jfgn+m=T- zVL@=K-~$mMs#c+gBEp=l!jD8mBw9tDiioPVis0%*OUG_6p&`ty2!c2h$r^H>f^>XqV@C3_HS=}?0=&T zgX+hg9}d?FShZez1Krhe*G{ib$J2mgDWd9o)1AYcuv|gAUuV0;9beo8E9J!Pe>8+a zI!|bupqVk&T;Io*{Tvt6JiqYSwq7qD&t?uQdj2&yxBCqe#N zA}F!V7LK-3^!3i{w7H%1I@B?mQajp_IM2bt>ado%*J>4__TO@wT|7{vBQbb|bAe~R zG688WP0e@T#@=-Ba^JD`0IkO!4pdKkC%ie?Vl^rgH*&uHCd?kE3CiidQ+lwR^s&*< zTk_9lQ_Pf@p-TJivc&NKu7j81;KhQ3tIQqnBV&xS$KjJkN5wCJ3*z=76@yu3nY}lu zef~bwKW$ap%V%#=9-N)LF{ukBv_4!{BA~P1Cvt~$T2vi^oXwo`qtGI>#$Oi9z_(WE z67dE*>fk%Ovh8S#HDiOFLWw2r!N38ZJ0;HGOe$$J)oaYRz*k>{(pPRk1YLQ!*6b3h z?8xT}hKaA?A(%dA81R3K033W8h<1{U;pCN zxAP9edFPSvnFN}3P^6a~Ubi}hC1?OY})pRAt zV@BK(e>cAFP1G&AT*Zk(1nzR)d)0g2L>JNC}xCvDlbGmTbA+>0c|-$~BWPZcX_9$i5P1fmanU%c+dnq0d_%2X7#S3Hwu zNDWO!gw~CT9XAZm3VW$zf67r?H7d$N zD-_Abojk88k@-Fsps*{x{o~21JI?b^EIp_$SMz7?M<1=-)WD-nQQZaj-iSacnuI{+{1LE!kgZg>O{0lEG?C)x%S$)F=>IM>9Y z7|AXjqZLf%s1bn_BMF6EmXBwNkn6OK`|N%o;xli!;?Cq4N~WaSx~IwHnn_bvbV9o= z>`%g|m2>J$^>HrKrBCx`EzRBn2b;iS1%}TlJ;%gQ-V!; zW=^uVVU+ASEJojcD1EJaaY1q|O;w2j@PYOF5I>7l!@nufa86yB=xo;a?HA>O#&`iO6y zU2|@Urw?&WX!7O;`hbn}F~*noq~pI*q^B{%cpvm?PWZ0QI^L%dkq+->Xn8f2O-D{t zu^yABmDLFLePbi+xYC{I8|Kx}5N+3*MJ7CT!)52nygNYuv2I^2OGL3B z$(q{#L@!i$7&A)`ll>2|$6qdTxTh^T#Mj5K1A$v zV_%cmf7i(+@`V{Ft(<0YNr)Bc4`p)sr}62EXeB-#y(t%|Q9rvk7|PYT`ff1aRPf)2 zt;zd_#^t{>M4br{_mmsx5r$mLEQp5}e@v1?g8{_H|B_vTc;86WRV<6D2CUnYA?bL# z6!-y0PNb-*buQ3q#D2lSFw>6{Ft{qqVv7)w=O>;N(Uak{&ojp~7zj^kuT=!{u>WyC z7yNJZG%VWePjc&)TDqc!ZOq~~rmr?DL#dlHIsyTEkFHO;xSd^@Jin70) z+ARD&@aMon+<#r4W+(Pzy-QsxM?!ATyN5Byz1-Ja!yy2pSMy`_KhhmB29(@b zTq&hdHOJ=SsA|#iyVYi}>PS{^EwdP@GtWyi)2tpobQ4rY5mhEM9hC|w3FLJtxOrqpoD zjJXXgCuf|}gIl@4JSuvUvy2qONqp`b8D+|;a6r^Hl3%LL%SSk!&TyZ{tXo1~K$nlv zo#monwUJeUBsZ{N|4!I1aV7PbwR;=qNbku^78Flb>e`F_l znq9@f2uOUyveOjphTY|GDK~i3v?mmm;-K}Rk87Gqnmk?jIBuMNk;5|2G3o4(t@Jdq3n(He!bOgWK~j2$2!OCnv8F?pG{m^^TguDS<|IS z^{jW_u2`;&KV-gQc?5i@SC%_7xdYcghp;5@__kPaf3s>WXE*4m&EgGNkml3p8I z;RrC~pz^Fr!JKesv+(Br%6O-kBF(sIxyOCRy{H@RG?>zp=#92dHyov@Km*VmfC6Ui zlY+93fQrV)OQ%j!-y~A!K^0X1e%tI%k9#(ZRG%^&rJ&YXRW$f4S$=!Gxc^k9QPAER zHeW9927WrZV<_rk=6OQ_t0aDXTP^Pv+wfxUj2M$Wqj*4*9QJ3 zz!OBdRP?4ZyZr2X1kHvG_-XLKFQZ3M+;xLE<2*}BP?gh*-hWFu>S1Qei#T;lJ96(d ze5B+>g4u>0)qCnup`42(`yxBqFVtgFPMUK6mud!phr*iyR{y_LGh9ss|5P)q8f>P6 zOwsYw4~;Fmg#Le2GnF%ilK)gQ+$}@Ou_i9(K=qxsawRg!WIzHZ%!ktdhic}37r;r4 zgMX!tKV^muN%vJFdCbDS{+=i;E@?waI$zwIN4&RLE4+m6W?Y;?u`O zy!#G1x@K_B;=qg(wNpcL619OJ#ti!?%ZDa0As&X(N5wJYTvQQ^K19eJM!wNjl=nR@wntEb?6v$P%+S$tcYZ&M?MY9d%Yj3wJ-4_vB%ts|LQdv7B$iE^Z z&mRokRuv3fP=TqPcI1u>l$z$kD-4i0!u`6oWw&qjT?Yxdy1ElK`*Oq%*ii}^%un4S z{Eq=lbhsNSEnmSDziHtntN9*wHMg(BP;6NxzzY z7w^K0G;cU?Tw@o8Il7>E{B*q+T~s@-*5&!v$C=A35KoA_m=s*7EJ=dXB=3mrxigdZ zhzxQ`n!Nlae}CXCBbQ;NhTg={Mz5#9$8?XegVun>Y%hkW>n;QXAnhzeotQ1;FjF?4+Z3 z+Mh`o z%g*Z*@CPH);=*F9;w$(mnA6uu;zu;hFEjF{%QfuC>E5@`F|pYXMGUjbG7pcWbJ-W8 zXC>28mG_=E@DMrRs{$j^2ZQWu$*Z?7AILMDDZ?xvEG(_0-4~kS1oDb6u-oF^%ythR zgtBaq32VtO>1D1?SBiIrjPy8c7xMG5Dk!)hH;{4S$M}Z7FF&Ns$MK@i^b-_eKy?R> zVIzOr)N;IN-1Oe+0uu}qMkzle?nQfrfoQwjAkSyQv%-qN7kLWf_#6*)kgTmCdLY=< z+YXS9IK!QblEUPZfaY`#ia)mNmD$|UJy#Wi$u$Q8l-BxII&+OL#-Wv)MAL-KH0nr9V)-qF{@JBL z1Y-wBN!^S*jCU>eF5B&=1H`fT(sbsDa(SZJsk->LwVXDhx1(%pNs2u8X8TI zMPfxdE?vAOn}LnXr;z(|#9}+MD-j!cZ|3aeL|2j3@%QQoOXOR&ro~JO9ku1T3EG^t zaP4~6M=v%@(7DB#j_Z!?aIUmZl})pNg`oIxvTNVKx4Y~_=O;fX-;2leQG>jIBO{-e zzXq`!YjKVbj99NKT(__bt~1_*fyXNkz_I^CIx| z3pdk|`@_*5a27`SaM^^L9yZqt4yRCP>5*7DNh!I~s7yY2S#SVSaxk9J7K(hi_a1T4 zoT~oSL|ia}kZ9Xe3!5Ll<`5Pz%FX8!JiQxwE#CIjxDj6mv*~NCfY`8VR0j4T-cL^9 zI?JstHJ$uGnx+aAF&GaU3RL=cmTu?%9pd-W(uIQG)1?II)s=70<5cq^h>#3IA*>sY zsaXaSOyq@OuRUz4Rq~*P!6Ok5!@;y%OmR^i!pda&jsnjXbz@wjcpT=#3D1QHf>9tU zwoNkj+)4uaoFD1@(`l3vZ%He=Mi$cy+`gIy_IX@gD=F0C(I&A_Rj!`17fEK4_pWk! za8%m@^)aQXVU7?ipbrHnyNR(ejWSi95q@kDE&5Vr zGj~_!d`(n=vM$8Cn54||FmALFsnUYYb%GfH`-ENwmURpjI@P|{axQ}&J)K1-^2%AV zvV0st0_RwBc1@o*1ZpqJYXU$R03|vY1{eQCn*A`o&n9;JrAhwf&mO<-Pa6JjWzhJ- z5@XzDu6O3OL|&r~RQngnO*XRs2$J`sT;Ub+0>0s3>FyQhT}Ld`LGP`CZgEGfEr5@a zc}HO@2E>`hul&Y0D9(BDjM)?JZ#y4Pau-8bCPvXrKKBX1{>lFVtaHmaj$lz$0-v*HQhdjJv=OA=u2Ysc=|FKGdQ^iRX4M)$> zFN(lw!w^c_uofoZ4}WJ|1|mKtB=_Js_o%DfZMe# z!r7M0IZ-iL2jV7a@bixNR{=U6Wq5qs@TC>|Iq^uZNb3@cXVw33+;Mw~+XnzNehePm zDT6lJ$TXKSvO(D_p3v*q?$tFr{Lg5}BS>IPX<$AJO&K<^S;`d9G6v4RV`&wC@{f=HearspEv$165Fm%q;|eDMW)(B zDmT*+VL{!3?ww8UPvsTPg5arP1f5vI+cEr993H%78(ur72(n9d{{i{81nCxhvMouO zp%<7j53po{9|_Y%;K2u)UVEKC%>cl2{$=2YpVDVJW;qkd1ooR4U6p~vwxf=}^Gmhay!-LU;=Pr4C zO<6c>+G}ojjGQj~F4v&@wJ_8XIhX#YGviA$r6eBwOEXj`OIypji8%Qk%Rw&3$Y7*FVo?eHB@>4}1 zIDKo+A*H5-C{9QVBJW-S*o`K9rj~o!`C3cRj>v@YBnaLz2VubHi0P(&;g7Eul2bOv z8Y>S-4$CQz2X{6FzhDE_3cl!Wf)*W@WA&$@SL=Wd@f&f8r%!({3-xch<*K= zCWGGzQYlB^ESEJPms6Y%e<29s`~~M)Mf@|hEU0DW&c)AN3N$FJpC+4CdvdOvU zgwsfJQy{XjHL%Y(pN#0-Ag&gDrC_+`D5WVH{sIq96br5pghk-xhEqqxarpnPP8AqS zRbuzG#DFSLFr&GDya>oA1KfxK)!~6nJAX zcIk4GG|jl>Mmfv*HlYat(OF7ZZv`eKC(M=A30AI7pk)JVnd97}E~OqPj0baZQQBq< z!fMfUN*LI?JC=t$5K*~@EvW33p7gvpFik80&30w}5vP_QtO+FnV}cAcfkowj$XIZO zUxud?$dS5U-sQRT93)}l-`ozAI>G9<21&)hys(5J9jXj-Z|W6-%I5guk6~qU#l2mO z);K6UIE;W3Q<>95rw~ZEazOaboX{Z6K4v@XUw}MMO@sdcmMoa=g(f8Z#6MsIHAiI> zW6NvN)S`3HhHXL_K~UK_d@$U;$p-Q@9&DZ#%<}?Xhz0V^71o9c-f(&fbP&MCAa!%` zj}!nhHn1ElK@bX-gay7p!P+T7wKh;^W?+&aEYJjrMyGKyAuADd1M`3etj)eUq2)Oe zdg$UH*z?-1R$~;;K(2!U4}NK?7K4U|yS@aT5PAT>acS^~G(Rs?B@9idDh(24dYe?( zO(AWjg(6IX!fSn~Sw2G)<_Z^%c#vuJWlitA=HAd(-~~4|l8zGwqd^n6oet}X%03^( zrY5K*6OhOiMr49i<%J}E^D;q~HVMK~u^`Scu>Az1f?x@J40L~!$KTmfc^*!LNh0BT z?#Ba9#R4UNgc4T*03l2BY2`Vu%w|7OjA-621Xg|Y+?s3+;$W`vRCM)vKbZjyW0APg#5??lS^NFw6674j8>@nvgUI0fAc!zg=&?3Cu3a0u}GFhW=ugw-6N zoG?SQo!W`QMvJRgufUr zr@YOq#6lit6o0*7VeRXT*9#OIwK`^jWUhvq@OVb7y?p_Ek1svl#9Q4;_uR9aKyrhR z2oGN3@e)ciOgukkx=PRD3@dp&{v*Vto2ZuxHxC)WbHoP_4%lRVxgYJUiR|@(7s~;6 zru8)adc$ehy0L_p`hM>*#lA3>Xz4*Ol$XWmYa_Q2rBNX?ynmNBpObd_s_80 zQY)t$`3tUv4e&R~oaLlpR|f(Dd^isXhlAWC(dA*}IphUH^n7{9{D^THUj9)+`mcy? zKFG!pE73WAmqpC&?Y|2^rzKu*?i|DmQ`5l2_4H;p6LPWR@ zc+g|2ImLroY@8p5*YM*vf8i*r3+^R;0|s{A5-i3EmuXS`42LUQAUk=RHAnGGw&8LM zfZyu?uWfk4v59T`tVcdviz#d%CsKeC+>NI^#FxkE)=WMV@FS~wfC>0C+>=K$%0h-x zIic}GF*ov`46B8d{{Zuj8IPr>aLp=wMH+F@&+3Q?qG%d)Sbkf5pT{x>o%~I{!XaaC zTLE{0DZgtujU0AW0SrDbN#F(2pg?st;lD!hX_^RHw^RNBdr5MkNGzyQt}q`TO#kx% z3kr~H19Q?tJq673e6v!ywhrAFESt^Sbz`T8YapOiHa&L7z$*+a*d z_V#OrOUAq%UA_(rXQmgf3;K3^JX4acdK@cxx)3!(-6>S!oWHEeO*0Rq>(jr98i%ivFzG5OXiR4bg)3`q$bb z(H*Sn(n{dkax4FGpW&K*!&gf#B=U~f{g66BhJmd5Rg1BiXHq5a!zx*C9zfvMamLoJfYb?NLXWz^_vf#q zCtfw-xhfAf72#?eE@lMFUyU*#z*yJgdobKm?t(~;P`>Vk`?0;0shYI6z{BU z%m?nblvj}Nhp-DO_y=5ELAh%q+dL9GwA?Z@3g@K2#T}95+(ra(dpbc~y zT!z>%O&oU>OBy_;iS9+z36;9r0S0w&|GsJol z7jQUDBZ1z?qM^ymdiR>>)vNmF*+cP>wXZxsNwtU+StfU7NhwxJ6CXzSHg8LxCp3x> zMFfqvt3ReUi|7jDHi&`ek`t*=BBi6{<7+J1doCPb-z^>xl6hi8Y(k>ov@VD}N)DC8 z*^1X%>Fiyvkri{fAfkIcCwQ_P)J$;zNos$utcfDtH~^n-enf<@7xUW) zB%F4i@^&uopCn~AZbHe(khz+P4O|-*Eh3PyBVMNm2b zI0!4xK_oEhCPeBp7quCVh^8)33Nwy*h)PlCE;ibd$quWuo8gcy6!m2DgQSud=}@Jm zr4K=}2o`UbNbFge0;Ocs5A>v^%{*xzw{9A{Hb)oJhc#Zv+pq+S4a98sw$ivVq9i!u zoO`*Bch^jy4V}VfbeE(KAoS9UmWfCEVqZ6agyz+9Co|*-48d zH|^TbfXS>XQvOG1_mz@aCasM?fL17)c@1&`q?C`s>_l1cW#%x^c4XiD!)ogjo|ruq zi6(?)+41l^^$}>`sj|zc6)Aar(0$mOApXmxJk$8OYh_{Juj8&KyKr@!aTRJmfjj*^ z7H&L#=d2lPW;?WFrg4Nt@QVntiOdfR9|7lV-hO6Zm_#&2B9<(G&qBeC0p1N9dr!IAKH+(FfHx#NMLMq}BPAMS&2VCn}%~ z^Bit{%@*efnX(NkjufJTh-9nb7Z`1awM!<&6xzpBO4X+h3hEEYzCWZFPbcLnL0qs8 zZ}Xt=k}aF4lkqiuYvS+^W?u@85^A&B4O`eGP{htO@=Yt4a7-2W0c`C6(h`DI#Ws0F z4i9Rn-{2!{#KIw?P^4rej{_!*VLF0XP!A6-ZVPR7m8Wnx*TOlvoL3NuzC{ZCOf@BEHKS%n&Z8ktvnyF z|Jmt|1JHHWFs&S9Ha+P9(gU$5l$Vt3d${Q9(a&|@@Uu7NwF}qxoFw{xYNKN^UG+09 z*8kdy)y|M1Q#NIo0 z#HhX6Dy?5j1hMxlYOiXIs-i1qQ4~edQmcxh+M)}2z5m1gJoi5Lp6@v_68F*JfH1A^ z0^6O{iEincj;!DFWi%r4a_|5~@>gbVp9t{cpKH))e1fB>Lyv}{MBMX)!~#RHAqGJc zeLV$uOejhSdcB_$2xYrTh@|5+VZ--Kk&`(pX`NRA^>@=3uBdq_-5nm$@f{iZE0tVE z>hFt_kEfld6E%df%~{Jz@~iJKyzzpMmsF zI~KMQ8+rC)dKR1%-KJfG>17anlN*EAy#B+3Dg1?hcaT{&A*Cdo|z^s>XP#H6wigw%tWQ6C^ zsUEqtL{DyPwFJGkS}p9BE*%g}ml%pR$e4l%MNU_4Y=#pBEBiS73k7i^Uvk4o7zLP) z2Jf#@xcfhjn_Z7|(q96VbQa`E9XhlIs`=jR>jAeue86 zB&`CuMBX*JQh>1FZjk>WnKq$s4Wy!;AS1fN@uJ|Tn=3!A($k?ste*9 z?BmW_D6?WhqzRUdi@3#sxna4bHmP(hZJ0q0GKJ3Q^(H8I3dpTNAfwBwEVUHjoO1EW zRm^O?Ma6o?i3;|1G_NIME~i&s4M;V#|4LR%&ri|t62GTHNMI=KM;QW!<00#}j(?=2 zSZ2^8+9+&yX&T_Gf? zmr3hieIEnflcDPgljexmT&2U|bpeKP29g&V=fRJ_MY=zesA2`M8fFU_+C3!X=g?zB zhTiH?FbMs)N~c4K*Tk*@*%vU4N>i*RzW>+NZ=@QLze!nX=+l!$uC|y!89T`OVu|{! zo}E}7rfxk+_w7{2t2UBv?=bVYO?$kx#)Enk1~FXhT8s>s{^sI-hT`xPmGab9+-j{< z#~r#o>&hL_ngD0dN(!j2@AaZLm4-WZ?Zv4D!~6Ovy>g!jh8+4*26g|zr;^@PXqhwy zy@doDDG5+;)*i@d0qe_&Nzl`NapA%MN2z@kdX+(BblT=t_KKjtM4`fharvzHh#K~M zn#-oTlYuwsECSY3X-E=)bR}9X49FY6U(s0 z*ok={BSU0(jZQ}cF!%8BHVR<&KQBew^NVm0XMCScP978k!r`TW-j$9U4_)z1Qfws2 z0Ej-016o|n{k)Ir8rPcGOq0o4UtioIiO$zlFA5vDk8Y|ogQz$cizKabLAw(TDfi73 zN}wYuxG6B#N0Kvx2t5hRSB4JObq^OF+O)idwnhF$$1DPM<x!d)IZ!D^;5S zcFs;TG?u?>mq=jHJkJJ;HC8?kg6OvKsb!a=!*jX!T5y4t@v$JTMuG3n8J^Z7o(rZ9 zV;Cm6Wfi$4Jvv}f z^7uG$>I!icfSUr+k=5;$$aLzNJnHdGjoddh9!iyD9q_bd?WUNR^~a_;5KW>vt5#C`lB?^oM`hNzxCy|6)Z5Qe=J+tdN*&ay zX=A}s#6z|>e%``R*2%`Y(xtHMcQA_uMYQu}^ zffP>BXp|%3UHTx$N(T=k)EpO=*c;Suv~9x|Vsp7p`y*Dy_l5eyCK~i~N7e3(^kq2h zt<;|B@_HKU#yI3MC>2U+^=cX%SCm|xsjXnm+IDN?GML)hOQkNP%ML)_PE`!T-^(3a ze2--g$gpV}sea_r*aBg6JWH`Yn`==d!!Yc!5E)M-4m?6bO`YTGfV^ZsdsVb%Ta?ZA zN+Cy^aKr| zh_9d%ponW>O9Hpp|hT~L^yO8@QYuE zRxT?QR|1+M6OJ=Z~}}K z)6&c4AMD}GU&_^5p5W39@C`#No)e5a6Ysqc3stGJl`)LK$vonjD_5u&_nozB8xMC7 zBfX(pr?u>@hWzf`0gY-N+9Woe(iM;Aw3y>JxDTG!5bnf|vifvk#=g&?L3kC2SK zFz+2?W&j(-!NGGcW1}(0NxvMZ|GUd z*PShyTe`5`a5$^;vijk}r;|2eEtxjwBA)J28H1l_behC^u-!5@_g(K~_2F6Z^*HeL zJooLZTd$t>b&pCz%$JUXx47mXJo)YZjO^68;N7S=E$cmT$!){Iurzwv?abOQ?A$LL zb~QriOM_Q8+t@~3-+E2YD$}H&M4n%y9OG4(r2jb~`VJppq0Ef8DX7AmpKGgobt@?K zDpAfqLC2p&=XZ0Z-o&ERyW(Eri3If;ahp`{6}2T%*8YVYwVhF4nj^Z@M&sX&@-;#3 zr0N9ZTLu((?6i0L_bzNVptqhy`;)l>8cMdA9d?Rnm?AWkG7PWQncx_P{dl8x;^seu z`UTW^1lETJHazl|dK7Rkl&|dN7gAC{y4q6vNI+wXOWpoed!4{MHLr-Hod%ttPRpPt z9zk!?cS}7wn*H#N!-3$<-H0FC#&R>F%t22Tg1XLwD6rr@pvEwRI-#4Ep`SfMw?ad|+zH*z z4Be>+-F+1L^=0VZWa$1%=)pngxAV|LSlD-=upe?^KXt;6EW?gH!cIcNe%%TCof-D0 zBJA&@uzxSZPA9|8R>IB?!l>tAG%Vo&;c%dQIPH~ikX1O?Gn_6goc?Y&BrBZZemJx> zobgpS)7x-nmay@aa29Gf>(#F?mIyZa2=*%x999vWo)KJO5!`npc(NjR??>>pM)1Fi z5GXn{oQ!A`iV%9g52yY*bR)6dKv)HHL=U|`all96;_6fOdb-F-_v>z3Bf~`XK zrn(~kRa{`+KL7@NUXCC6Lm8VDyx#(OUKv(=zX|ieG8^!MAtdKqwF~*-)Qq&$&pWSv=axmJU8O<;UZ0>A01^xFd&q~uUp@w~ zY#3i!3i4zMxbma$MQS)TvaS1s4NsH_{DmxL^n|exhei_IYR!c z?EyoPd)h?ck*?KsQdksqveSapcM^2e%NQsU|(SR%EuDgob)XpOnmVnK(qPyL8a7F$;!teIR=*8pT zD9WE#(SLc0|Meu_>K=(p@dacx#Iw=-@%-e6JUzL2X~FZ`iL?l@dG24hq^zdzO$Av1 z3-51D`LAJD$V|mCV)~F*BU4u;AI?_(dXijm^GnRTwD?qcMoRXrs4pjeeb&3zZ_9}o zJ)H|wz7#|=^Y%%%9Q8>uzLPboB0l=K4j?lkdd>DfRstYYo>oP*G}?WGyBIB|dDb&R zBT6Q=VFql6fniR6G#`mB2V06&0u3dJ`K{|j;ZMd@0H{_P0uQW6zCks(fxUnF+Glv^oy`w%fmW|OpSi9Q5nzy!~y)!V;U4rc?M z6}xGB?phj#r!928L)Hd2)yv zQEx0DHft`<%F>Y2#`XuML|#-kcK4_MM5*-6sLI0}aG{b@eZG%gEoiQlGY)oI`o31)6K#*fkrrV|B-P?=vXfN%)C-*Cc5+M#19?iOJ^QFZEfc!IJjh; zdiL0fp0hh?$SAOYzSx@1>!vj=`m@@H3+#c_z{SOv0*1ea`&7n$gV$m%*u!WMFm|eP z+1e+--(9x?IjbANDd!^=r(jE=xuqk-A;Qk@dz@Ba*Enbc~4dGn5dCPnvNo9j;sj0!9)pvmlD zr@_N_#<)F-URGy__>$d{;0*&6{65Vl@WOq8Gx^fVI zGe;PNoF6yOKI3`zZ}z3?WW7V=s$$Yy?#}$}ox4<(?9k+_YTqA;YDUlYo@JcXc#>oR zLp&pVYbDl6a?J~DU^JDzIb1NmkK{6aMlQW8(=oD=Xm-tp&tV_-z6hsv5=`imdGdUD zPK5yrB7HwLWu;X@j(%Gi<6FE=NNIX8EbgV=MpCI~u!9 z4?_PkU^^FWJfQ8@^?8RaZS<0)c8sQ&&B^lsE;oF{9tZZ&=^cK@vacS)v|ZT~%VbTF z^wK;s#}vL?kj+KowDJyZHRK+0#+Bds-Xgoy*MzVk45|t12cxIwQ?%Xhhyh< zQj8epEDk4@w#6|~O5a$v44{w3!Hn32?2KOUtFKInR*ywf=NyR%`OMN{W{+nKGe2wo zn@9xz?RdvI(UwMtn85j*YxFj*^BZXFNp_SxG8b%P7RdXuQsQ@%}WlDe&6LHJ`Y-{bHsdH z;ftUdpo=9l>K>JULWR@Ha2S$QC~2)wVKQ31DjMmIdr+q6;<;Q^4R5glEmsxRn&Zo@#(n!Yz0BNI`B ztGCx^#hp!QUfT9EIb7g{ZRRE|;0YsfC=Uy)Yyil|#9|dLympUvVCo;TjT;`xe$;;`Zg1q%*G-BpcoG0AaWgKm56|P5VX6}Ud3)^!h=ehwld|il z)xX_KO!Th+(95jmAsN(PDPhQf+Q>J*$gho6qE;!FgxyMvKdIBVz8hYl6@c7TX_|hs&nRnu#AYwZ z&foaxQR|&EY5V{v-$h9m(p5&hSHY2&pYj!Y{Ljg&|Ht@;8wOXD2jeYlJ~w^7QF%{t zlzEP)QsVyad@zrR$gr9ihKnY@Ry7sC9aTBjA03Wrs0F~7zDoS^LLhO)B2VkN!S|p^ zxGM;Fia&-H!ITZuDS)_IEK;rR1{~2G`h2 zNpcz5$7ZK5`W%TR8{|E zkWu_b{qVxD2wmF-m*MNrgf>Ctmv=#^8y_Qqt|zSS=HaD6Uy_`l>ftgOa3D7wSxV-T z({&9A_%%&y9?n7xgObIy-iKkziIh|o^#;|n&?ECxLmk1o1b>p|R zEf+3%TNBw^3TjkPEk;xNBtvu@6|yL$(nEvxuv_XBgO$!&l(XAGTjFY!B1C%qh}x$; znZSK1DaIcb3+YUsUv)3(rlW5Fbf|cYf+oNxfU5f&rtGZ8I^UibwQ$RFA<1v*mU^$}Lm6+XYI+Ws!}R zbPBe_n(McZ^L1s1qpazC{|(6=EdzL~Qzt-wQXND4WamtJRKZvPBcem2O?@a<@1=M# zayZY`v;(eFE*`CbJvFWUPY_Y@@q;WU`NNUN^)?4zGM8N(p4{waneyi5&#Jz~wGFQJ zyN2%5Rb8y)`Gmf)Bhqv>iUP@x7g%Iq88>|y?JDO>;T+=li=M=obN%%)%iUyr;>>ah z%u1nonfYHiA7S}Y|=H^$tZz5$(`}jPBPG3m$wZQ8Vx;CH>NLR%V>KyORDL=LjX-xN8F7%do?qIjU>kJEq-<6m_XRB^b*cnL_xN={y$i<(uyv;dznB;zW!fx7lVsU&@ zwMJP-4DU>s8imv;keDV&$>9K&=R=E6$PL{v+cpd@d3qK-&T(3Kq{Xs4_yHLL7CEQw zD9KJrr;^8fxYt&PjviWDpyC|CrdyQp8aMj2)A<8;5!hb?uz&n>2bw3cK7l#Lx5q2Z zt7h!eOX-0raP^f>HgO6=TtOzTS+g!4;;?_UiR$yN;E~FMA(k5%lQ*u%L9BIi7v5gM z1DNBd?TL?ICQ#+0~m!TIVJYvOUs8 z%*c&{-G`=dF6+j2BdT?I9IL3k9N9H-?0$Uwl>#GIR*ynBDScY_Rc-Oi9J3P#m>EYM z<)ssiQK^vjr9f3Yy>Wt!>g1zA(TSc!ttv+5&OwfGQzn$iFsmS#97kIQy$aqN+@Zc0v5*1KBpmKWZ*5t~e z^K?kj?qKPcDTz!ls%n{;fX7)n_KL$sWD+Ss#QsPS%*bQv#55g$Iy$RGZ;ryqI?^{V z>7t?7i(L3mI7%wHTdiL@SM(()2L)NT^<~CbxDdIw2J(%&!&_U?oY*_im zDQRnFgng3ctx$W|E<#Qy7VfgX;Su|P6qnY>#D}1- zH^lFG`@P;G4;IwWrZY-9EwOhu(s{Agrj|T78a=d%Kw%9s5Rpt({0%B zKK+gLZmG1|8^XJ>6F9vHOYUO|$l6y7O@3UaMxnySC1 z$uRyth5mlg{h5=79j+=Ia0d4apK93+6F#roP_B0lL4SX;`e^o?*k=R}ENB`&(lr~e z8e#~qcw0YoTImU6HU@ARU03;Jl25}m`wk=ZRc|2^XwBZ) zRkNbYb-9>#!x%ti9@%DK=rHym;yC5me+D)PCwqnUl2Pi#RV20XZO=574*SY=)pXj*QR4HorA>= zbAqNbCZH;TOxvz6tBPF_#TKNb3tg{siWuxS&)M*<4)ASDKJ4`Qo_*!(wMM>V3I5F= z)%(n^jjX?fuch1>hZw0zEpw$1?1!|_CUm5Y%54aD6`Fma=CeeX#9G(w)AUR@a8Tz zE&F=5!+0c64sr1}rdzMTvEP{BPmsv8@!*V2!=C{5#i8M3iOCcidhF6xVBHcMO=cUjstC9b?m0Of~GsJOWxP(QX4e|0se5~l( zJEJo>dvmkS*ttr?iCokgV_f5ifbopkY}r`&O9SMtKD0O5>bCWiS;6=?KMv>owNGb` z3fCN#M>sFGm)#o^d{cT}5(hJ6$-BrKmbVv@OGgHW75oyOm(HaOM-v?jP1cd{$=Egu zZ=+zsso#p1uxO}RP88$SV}1xDl?0TIE44p1DdX1z`0?n3Sd4pYQHGsvzTx!;-KXF= zTH=QAbu`y%x3tnebU^(lKYR3NbK45u%|u+{t@Kdr>S-L-nfHnHFkY~*%dHC&cn{Ls zr|IXiGTr9gLC3+;4=~0JHP*;O^-rI^8__g&T0H1{4i>eFMtB<^b$9RV3xP@3UBh$} z;g{}D7Bfq}9*HB>gq-U?n@%zMrMRq>K0`6xavLQ)=i~b|dZQS*^xMnoFeR>#n@<<3 z%RV_4u0IiNFp(!K&AYKq2jvypO@7m8Usg+1v5*}h{-HS@5 zJGF2srSl0JzUt8%&I_e_dlq}_9Sv#o<(%`N@@g)QB5MEGEV(8lxd%x8HbI&*A-YdF zVEKK}!BMU?=*LB-mIx%QZ_uL?W@09KeN{d(Gt-d%cU+Q6`TIYfi6B1<(R{+XV{BqO z1RzXr+yx=+#ME5F_|YE*&Z?6ScIuKB#LX4LcTN@S%Hk6;6V)e$91aXfG7>aEaP66lt zoX^eJAYqnI@ezrcxQ{!LAa5omzrA#++&jVT(RT$22m7j09;aU0)N7}T%JFNwH!qvj zo6xTFI5WUiu|GRLb?!hIz)dk&!53}62{Y(sWEhH%K;5?jUn!9ewZmt4Hw2K_OFXOT=4VtY@A;RYx;JlR6nD zzha)ib&MWgokI-`Hca0kS;>yT1faJ+a=iLH{&v_+aYexHL(k7EuXfUHzT75#b{sJj zhl$yay>gpgSuun5IHv=4c$UA4UK+_W0yErKGC$>l?a}!8&MWR?x+W&Rq8VbDxta6x zIZBlN%}B#&uwW=uBfM|AiZ=1C;5oXIl4wkq$bmCY(TnecirR#qdg~@Y(&9Nx%NFBv zi2ZjtG|Ulthy)^&U0X{n5129SE|t7zY@7>D4Vv^n`#5kVs%%YJOCH{=uZ4S}Gt4 zq|a?zsETJIPzaFR@i7|!IGo0tT<#xdEtx#4;opF;PMA<&a<1H!q&ra@DR4AjcOHI+En*qT9lL-V-{=ow$TjE{{$nNH!^n9Mm<3 zttj_@p`K4OP-B%)sx?xRHlnuqUmUxO-psQ-rmFKh#whZ_4Y2?(%Jq}tX07_ zE4D7{{{0j6se>;5kSo(teP+Gsr34Zj9lhoYOmnF+7Ap`kd-hbk{Qt11W#&aMLz`I~o+u5`l_d8AFMz`0{NE70Tm56@I(PxLr;_WVH zT30KV(lN4EwAP%)+3LG(Q1MgH47j6G@yTf|ZDT|&tCM=G447>8bl23XOs`7`^hLT2 z@ab#5ZRHD8rd-IyLlkNTi#=zKfSBP9I3y+lqnPe3&TgsoELwyGru#5@F})J)GS8im z@)0a{$th(w&t;Y@e^WNwkH65z#I&dh7JunGP=Viz9@op#7oQ_8wuP5QAP<=q-;O(X z-|-v059rV8gnoIN)WsM+v-Cv|Pnb&5@m1f>&*yruSm0}Gs@WR>Vd9LOp+R3){MXD* zE6|abH4at#TXt3DPe7HZLmWVK45osnwgC$*?aJuqc9d_5Ns-3$>bK0x%9W-po8h&z z15NCDdP`64p8u4VVhpM~-)iezY~w;gE+^MGB#mH2BkJH4B8H2jmNG7Du^&tc6!0iHimkA_ zXHU>USH9e~NcdiO|I_TZJAXcSQmN-fgoTI2_TN++u zz!uY_gCyZ8W`s~=Ring@ONEBPikB^U9ni>Jnw`8+R!zmrO%nE+?mQUDl?{jea@ z`C|$3G3~;=fmH^d^wO?pdRsp$n*MzVhShsn+Zlmn+@@xCMI)^DmA%o|Mxwt<}ym z$m^*Cl4F&2B2Af|OZhdEEi5{+Zry&q%yp>?@Z90*&7ZY$+Bd^w=~jZpN*^PC77xgo zeomiQK)*d$vZFj)$FJR77^wLAYMNbnG+One{;)3Z`B@Oj@1Iu;*x&ZqutMZBjt^pX zH*XY?Vr~<`=J>> z1-5m$|4yZN9G&sW`u$iu8n(7{41M=ujhi!i{9finY2LF}g4A!lXQn^)O=YGQ_MSd} z!5n%pJLu7eB5V{KfR<#jY$nzIclQU{`6$@ zR`~;=llG#sy#UYKA3h2F>XFp=dPU>U&{v_~eZRK8C3?ng2n+wodv)ta$*0&ap22_K z7M&k8dH(m+Q~2NF`}32YPycBuV z`|p!5b>ZyfX@X_!DOH$;)|!Ss{I6RU4a?!tN!UJXHh_N$Aj}FBvj)DCKe7%3%Cpic zAMPDyo^V;y@@7$Wrnc2tw~Y@Ms4Fao=GGvmY;cd2@5)6sqV2mD>-Oo}KR&F>gv0%C z>$BgJz<*|&hSp%$EIP5!?WFKU(^ha%HUl-NEjDu_S^=ChwIF`*=Y+-ksC8GzlsZNR zG#zZkg;q4SEp%E#mF1Vmnfujb8i|wkJgZDYnRHc+3}3B(4Te2SJv?b7X#kTR@Q7XV z*`=o+ug4M;g4@b?HbCbD*4t*va>LN9)*nm;th#c4Ii^?IntH3lkLkuftxvUVF{+&y zcvFm7UNvp75;?M_(3l9q3^U9945qV8&;&z|oN+&0=)F^PJZyNxxE-!&cm$I8ojVRw zwQgc#W#w*HY-|BK=Dea(zCJT-^@aeAwua=jwm{(+$YCPgYuKiQS-sZe{a!^jc|-V> ze$Mv&i%~uS-KI2s}eZoK0{U#`8fIrCrxz$-dcZ8 zX-1JR2{vsZ)Fj6fa=;_c$|K$W6t54tlgn_2&9@>%sl^#Fv}+^Vem+X#+K`P8icJbqSSf3%&HXabMM5+$+C`| z$tCuRLdMj{FRI#pzI%HQt>3(r0bAMcqqRjlLf=X5v;aravrC~lM^^0}%Fhla(;YEw zl+;h1{?hfN7luP&V`Dw(d!te7;Ov^Oxz`_!5cYrUTSiSx^`{t5Z$xVmQ>Ow@eEr&7 z2{K9gN?7p^j~LiJ6_^J`M}MRtDMS9T5xPGux$>qRwo@MB;^++@K4yK0`eU^$dX#=B zZpRg&u%xomoNx446b}-6+BVpFuH5J^6(=t9_PMfb$-95MIi5ryLxO=-#$#GOH0th* z7!_^Mh+Cyw0n;0fSejPKK!zOrCgmr9#O;8P+IQMXc>3lL$+5Z5BsQYVVDu8Fg;ur9 z`PGY`ft*|G-7q>QqLM00;NhJP?VEley#e{sY7ivFtyRp;R!{%0x1s>;)GDW;MUdnH z5#{lyWecgcE}^_?TI@zV6t5FMt5#tK^$x2#>CDx`PEf%F+aEua4sPfyt*fj(z~`rl zLQ?eFIE`emo~#)nk8PJmGB0I{se%<%5`J_xnz(cUgMvks^^zbSmFUArRi0|(S2^(& zm2RzJe^RB~ixZJl8UJTdcurAoiv+wzpzTlhu>$=8KYcO{n14VnyeoiuQ_tFz z)-!g!gyuk)Zxp#sM{g*CjV4~fT+C4ThjK;8efJRJ=+w|>YE22|**UynGxrMe*|k!plUkZP$8_=A-aQQ zfZ-%wsW33I6RmsX!>{}~P8_M`NU&`zym5cr0=LTgkX4326HqnuNR(D43X>}*u3Y%p zT@Q;m1jd>v`Ru_FA8Sm`y}~xu6|BV?38V5jx?6Ay)b^&Fal7ATV=_k6Qm*Do63u@t zT&8S({Qqe6W3DF@IB@Q>;k=Z?&*qh7NCBLFiC2d-Q`0?~{0v>bgQD^7I|sr?4nB+A z@`#$m4_nU)>q3l!5TsSj69fIW;qm5XKht5!gec|M;VabVMF=|iq|36yrZdRrIi$U2 z*7uHxh2OXN2d_BNC`J8NYNORQ<}?b{i|5XvCu~W2RAR1I<3DdmRGio+iQIJVzup%t zH_&9ZCo1k_%b$?TtH?DiN*mN(eV1Bja1=1Wh0SdyiaS#vz^){Ue>BTeWBRc&t{f@+ zA9MO|rsqZF|2|97FQ}*%c=iS*J68ghqYm{iAMWi#)pOfFa z@|8nDm$*x927z18%7k#Mq`hr*PPeSU<1^VKqZ*la!_l--^5lu7^-kQW4N~?uF=H5G z*qtgRk#p_%5-yL)B6`3Q8KzvLxnr02ZP09cC!LDg?BN@%-s`B%1bbDD>Xz2TNTgo! z#{~KieeBYJgrtVQMKU_d4!y|cUSG!%V)Hhq7N&5*x(1D{jciRbGP;B?5=(Dr0xqaO zsEjt)8fbfKpdV*yOI}~C?G%3-BWa4sH88No5=C?h0Tg4&TcGepT&v_sfI-VM9O{8% zF>r)%RcKGx@>-%qSCWqlg2-iSk|ZPGV5TxFUF?{)niqg%l7Cy%EYet`1$33`#8@yB zm!|MbTmLo8i@AZzRNtdfAVuGw4^=j=mg?y%xR?}IioD=%aOr5v-d4@I-o7zt_E^8D zfv%fcXbgqd;@-=}{eNs6GHF&-{;+v4K zqYFP@41YAW%+ER}#fTGf1Hj9lb!|9fz5R8~uRlD!$%@@@4Xa&j#>F3gIDe7Lo&n{yy zYXaaPSz#9~qs~@jH(kXE>bKOQMgx~ld^hY0+bO8Wef)*L#nth~M*KvNub>B4q}A4m zo)9f;VF4Y8ps-q~B5u-m$A0Q930?iq+R_&laf;c;E`PWMDJ3N;VX&)oL7Qd+hT@v9 zlauxCDbDyRZ%SO+s}80<**1+}Q-icNhsTI)T?6totm7<+Q>ZFG7q$eU`Nr~U`b8QIfS2T{`?i@7tM(LY!adTLAjmx_1 z&A?3M#HSrg+GEF#5<@APX_dwO@4?SnbuXNTjVizELTHABT8VBqB&uTcGnae|g2W1Y zF70zW7DS7hMGrWb>bC(z^^>8T7g58*EJy~74$ikHLK??ArhIVXLZkn%HXxe zdeu-=559GpY%`LQY#Raj6PV|f#ADok(1;@-;RtQ?0SeKzaRtnhvZt_ME$;CgV&z{) zS>3YWvNBK_boMfMi^5i8FYhVPGCxEJ-?^AP9?XO>b6Ahiln!sfF5U&5jSoLf53oNF z74ImJpM0V2tPl~Bt(%y8?H_b!wWNPK?b6VCG11v)sz))2L#4YEX$plpM&ToK;y)$o zX|nd^^gSMgNmC1WY^&qLMFXx#YE2(s0$0Uco17mdto2d!O-p)#cb5Ji)1#Mpt91WiDV^AsHbYzezpwPxXSyugIr(tslWCyQ<#5`Eb zt(QC`6?EAxJeqKQ&eCVU3XPE%RJN`%ugan$lS~Syp-N1#7|hifT;nBLp2TTEH$>3` z3+Cqw7jY7Xff;w)H`g%KRT0q`Y6~WTJk@UEQeN<%@h1jXU)*Ph+h+{^l|-1Se^ruD zs(*E=+?an&rVccqhF4KB&$FSgZrdj=H>tc@bQy<_aVCt+u&U#?iY##!y88d!A=EjS zAqOkJFzh=g3xv2g`Fy&fNgmZ>xJ4l4>Q)vczr)F~MdQE&+Q~&BwbPintmkLc$8!vB zei&U}rs$$I{LTNIxqKAQ0~UM)!^x~q6g#ep+umZ%FSD}-cX4BxQ%8{jy#B6;3}Lvv z^BqZD|Jhcg%yWed@WFAJtvbnm<|-CrpnL7LZFotrq+77CUKfK)gJ*lse4me}_Mk1( zO;wdm){h?V1M{Z*tQFmH27R*w+e#=B{uZLM!;8plv=o!I+?nemues?yCDYVZ(mp?7 zFX>T!zJUzm+7MxTilihJJm`JYv-jN1DR?2!%Z@R(8OIDyY`;L-oD$wTU}#t9;0A(b zT?I4?Htlshu6imhvy^*E(qBaV7se!Q!NEhgy2YOXi&GGEgjwH$dLz|wzUK>{+21B# zu~Ck8%|V>k9b9>{tNmR+dpr8@6p8g$nv^fjV>Rp{qPf_kMM)3wIn>5$Ro~lsC9O|( z7jY7TME$ln)q9BA7#_pOx(y!^VEwvmHSOO0?a_!#A2k3Jv`-2=~n)y@Q1<6 z(jE~iPegfhwm`_hnH~{unW7t8B4`=zk?V!6WxTZ@o9uUKo5QMFzOs^@4CY&)qwL2j zzEO9SM+74}xr*?(eX;H*T;=lLrae)p}{QUx)eJ$hfidNBo3hLYEGQ6SF5*fhFB28MVlUbV%+@_LdSLcrZIygx^# zHZq8<|Bp^dJre;OfYf5rzU{3&*|{lO3C~R3Gpf;;+cy7G${4~oBc5=2&EoKT_CJUb za?~J9TCuUDP~{po*q>9nSahP8WXcD!$glDE>u)1Qp6_hF@h~{#Yfb&}741HQ#-uT0 z>z~I+u5hxo4ezLDCr+%}&{~|2Sr*Skf`acJcrg$ zDCkFCIvvTy!QFTq(tQVkb`QPJ5afBBvi?;yw$xV4g#?z>YnOcbvAoGh;me8TzUa%$ z4E@S~Kw%>jw($>tkQ7t4!(#swskd*HuicuAGO_pe68m=JOPn4U(Y|1GBmYDrPoc({ z$VF58L4%U#A*OvpFp~XXhQd-J45n};U*VdiyQ zfD;JE1T>3=FX#>)ae=w-xB0YpHB*0-@Cl5a$IK8#^^LK%1*i-%moiz`4lR66rnDq~ zCl(a)_{qPP6jZVFH{3nj7oj~<_bU)KHjd9>r*Hl1Vwgxbz8$Y}k)@!N|8XfK)0dLk z9x4Rp@a$f0u18wGz`OPRvwaLfI~Az9hO0P$tE%Qon+4xi{EL`nLTWny*_T&Ya?Ee! z=MG^acb5G=rL2tnbH0AWHe#4RZ}zDzwD4NS{VX0~AKK#wN5=@Q?yzN&IelDkAX7;4 z)wFxg{#Vj|D+)7fP*?hT{wGze)MZ*Gf2`&|Kwl8 z0?4S5YO?E^TBguA-+zB6Ub!WBzs}(QO>}kFpYkj_RlA+iAE#Asc?$AJ16zT7=$+WI+SB8*ELcwK%Pk8^bxriP5@U5<<=kzqvrncMQs3sCe#h5c z3-quZolUh4)DDXO!;>FibJ}aaufKxUKlfKnqU7DDw@psImFuJbjBz*u)JS(ZvI58T z_NIH;IFCIA^C&0O+zMstPW8w}+EYaKXrznUpxve85CzF8=h$)g^R-RJ8u5Yw_jO$2 ztY1%H$=}6qY15=M(;kM9j$-%DPC}Rs!KiV$!FWM)CUwYFE^}SxvMlBVB=bp}hPK3AR_GCfpG`JTv}KFF1V*)KyMrYp;8qY_bJ?>x2BOHs$2FTmr_Z=G?+)15-R zP`f~JjVEF)T(Pwxb~odlxYluMnj!{H?vlcZQU3~OcxhiJaYG?NrWG>Aqy+D9R}*=w zKp~;9dZW3o=eWnsxJqOS*K=0RpQsCzKs%qz7>{5K(iDKAesp6=XO!&)PUM)gwDJ>0 z`JD{9SfbFB`PV1Nke%b^JrspX^ivm z!W1QQX_fr|jI-!p=XBr&p_ue@Q#uMCwCFD1@_Dop^bR!{e{d!$TUZoaO16iQ8#&CMq(dyA$v?@oFms=#lCSKu!Bp&5RrM398C>Ie`d zh^bA_a20%2JuE4f-pD`-dMx&Eizk(wSV?Pduu0)k-}OAFFe!{vkk|5-`<0=r_1L67 zkr<}*_2z@WY>$P%e!K9v!Z#deKIF-kzDvm9rH*oazHRAXWB(6G~10DE4<=% zW(Z}aXo+_-bT}%r<@tWHjwD^UX=fCV3O!Vcanlo5#vW){#%Fu}nvPV`t2oGU+bcQt zFgvtP%e<}6-ArEC$XVE2$MT+{oEq6e!Rh#?{&FwFqe+Kq@Ig7FTIl$cUwx2t=Oh{j zJa1oydMFw{w1fk#ir%yIyig7-{4TEjyQQ(9qzq2+S64GJ$V+a%+S`aCgnRrEk5 zTEppF?{jHiwHXaneti`ON(~_qA!Ex8Nk8ODCIGNKb^9t}QnK%1u4r3%Rdh1`Va0V# zAFIo=ITjQ6xWj<0SO=ZCO5Lo68Ymv(R+qM;@=HT9X{LmStDbgwRpiJ^3c-SY%6J*5WScf#@n`c^awDI!2iQk3 z@1sXINZs2z(QDZMEX`ExV`IvT3P`NlM};8j2J_?=#|LRvozLZg@wp;3=Tq5UkI!r$ z_@!h9bNdw7fI93LQ|Y?rQ+@{1nNO>{JYjJ5L#w1!C2gZUvlOFM+8O@iIW|>CqBdATjzWjL;hNEBEl~2 z-CM*^TZ#1H)GM~x;!MJZgD@^t$*kG2widVrMydD*D;Smcux%W1fm1Y#Ze4x{iP98?mIKf^Qmi zdf~)Vau;8Q+d6#Fp?sAP&8_^3cmX zzt>SdpfvZJLc>H?3D-c#kd3JMK*tNBu~FkUn6l{ud>QP9pZASfjOnpde9CW+OJ9vW zUUtzt2lF2cD%f@w+TBJx?SJ!&g|pVm?j2yjhrHlaw}{(*z7Y@gZe!}YYJNve(=6<5 z@ryge&e;L2@Zmd^EhaDVEB*cSZ$k{hwwUvGH1Cu@`PC%rCD-x0ufKEXe*OsOb_c1w zrST3?JimJ+wL2qfL{PqUHvcZr>+Y$4yGK5aK$}{O?R?%w^+ycdDbTvXmQY@=yBj@l z_p`*np6b|lyRos4BVXX7(zbUOeaHUVbr~9Q2L6j0IOrtuzz1_nNB?x*9%?D7-U1a7Q0@W#Sm(o~GJ8 zZTow=S4QyB_a=uYcI;S&yZ25aCQa?f$5baRx}pra1|O(S7#DOMkDfe*m~vJddi|-% z1yT1S^p0ij;#*sE3;*1 zr%b0JZ;-Mgvgbxn2BV);JHq{_2fwFKSEeUn%@>h!rEE<*F(h z72O$?eVJ(wo+0qjIBJ)D`KKrG{9Wgkf_$`uyH0k@|LRy+fB3Lv#N&Lycu4i!@Oboo zcDcP9axJvg-$-!+QFATD)aS4}*3KvB5B_EI1ExXi{l}@sM~h3C=M(#d)!ii-zRvCu zqlgFAsi(MYZiQY+&!6^p zt0W%7Nu{f%)gf9{hrA^bmUE6;g^LKzh%T3UCiJe+jXO5tVvp`er`{Vs^UNy}-qmFo z!aXi8;b1-uPjilCl6r4?&&Y%VBQ@9dNPiblKsIo+3 z@;iCkm`&p%-0LR?+nHF0=N0OA>E*fyy&cx-Hrc%$I;S6;kH*U{J^$sq(uwQUgQuLj zJ+GQKT4EfnZ(xJ!VO!2mY4`dnmr*m?K(Jo?#a}1OaD~vF&Yh3fFPPvn#XThDPbopA zx_q_FJxS05^UPkUV-HNlo1O)$D@TbBTxg;^&U`NSy8{$($KXo0SBnuc+gRnU?YSj@ zd~eX)3&EFx!xsxKKU=c9Ok43v;M`x_NZWOwX2z)LuVIU14FRu6^c%Nu%{Ea$DVS$C z(j&1lW?fYJ@!91E3U{%psogKK#jC*qNgtR>avI-`OMWvIH_dc)nytoVgk+%d zf?uZiIA7@R2;G==Fa{leqN$TylQ86~eoaC`V}$B=a&f~%*wN?hOi*2&;)ZwZNzKc$8kv6S^Pm7KYrjHir)Vu^_x;{iS)P6w+%$A!Yd!z@L!pk zc75>hthG+$gS$qVI5)2Gvr!j0YF7j56Vls%2A@9LVf+DC7eD^d$GxptI?9;6mZWJL zKVBh4QoU{Va`oG}tm&|qh3m8Yk3Sz5$~0A(ETXTj*G(Kq=};1jj=%bqe$2amqy9Be z4)71cKD#xR9(xTHu49O7OY_9?cv^>hp-`UGEa@C!4}Ltv0Q5#)&b%sJ1IJ^3DJv=+ zqoxN6)&C;!b#wY&T}M3?nv@watq=TiTqhn4^nWGQ5cB<-?u^HgZD_snako|Tolm?< z1YKd3q^uzSQmmLR_(opkufnOnT+!A~mF2$JubBep&uE44yp~RZDjnHS_Oiak0}v4f z*KerUq#u?%->#*bf)xJs;wV}@Vk!5$^yw-0iIbe;op1Upe8D2ilU#gw>xs^qT87vL zQuEdNT^q}r%&Sve;??K4`1Vz2b>+u-cF!tbj=MDX;<&7qYH_DYW4h>W3ySRJ>QpNC zJ?41$3DpQ^Y48bE^lIXup)+1bK5<+jf)}I8LOt4V^!dj}bYr{Q#O!jKfOV9Oh5B5gWRSHs>w~D$r5KBUg<-VG_mg!3z9}2CzVTNa<{DsW9?VW_2$&U%DZ0_b%HU z&rMfDzPO@1shlqbwsYCH*;;!Bv!r)5RWqi_oA@vM!;?A?Y=P0?Gjj9Rlr=4qx*sQB z`&fQXj8tB^W@`3%tyHH4B84|-MXDk7Cz`D@8ZC;XPcNIGz5eqzzg8eK+T;Ba7V}Ir{Lgi`8!7;P!VkNq#q@!3)RIzyCF5Q_Xn=dAn zU9~#iV4f>v639V{I#$^1f1cHzWY-(UBU;DvH`Qu@sXql{jmyJt+%-eUz8x2MgzXb7 z{dAPqyXISzz0(~efD@q@gg1yW8^iBN*@hDbp`OTpvT<5TDK#k+DBH zSk;qxaq44nvVl35(StF(2In%A=dLs693axuBZUw};_ucQH_p7weNv>O-Z%dt_?J!Z zFSp&Blbq+fjeT4W$u7(%H({G3ys#^GC+*l}p^pPUyUjbg`33(%T7C&~{bM`6OoFI) z(+_OjjPCzZnjM|9%_oGm8q|9X6gl)ZYUWl?H2N58+bEaq%SLvJ>blI!J>S4|zG3|r zn>!_U%qOC&z~dWK*Ud`H^9;jQuG(fG95d*j35MPFsM;sF9II~o`{CC1uyA)HqRUQW zRPfL5FNPn?{V2nkXt6xr0TC_~k?+`_R$!ry#_S~eWL~|I*D1a3iH@%Pn+>4i_uDquy7%7dV>NWq0g$aOEK*7#()y=K6+j-`lfp2I*CpOx3w;x{J7UxhL@ zuQ=g+B+DKZsTkN0tOcYZU~WR9$~gJvSI-oi6-iHK-=MAU%3=EBzNlnpj{R5-JDw7r zQ=ugs;aQFjK6T#vr5PT4k+1B6dV;BRgmz{(WVu->NWu#uski{$5^4$evyktT$=!@S z{F|4>zEUb;nTaUtepzdT%{>)YDlhD?Zf+e1krPzP|LMeeHOIpdN!+ezk=xy`*?;0cI`bVj|SD^#ACH68B#}6qcF~p z6FocrIBR3i=^LIZNBp3DU64;F<5E35z z^uycu($#d~BZ)*4Yf8WNeE`v85v_;G`?!0{>cr2b;>$e2Sdh5>i2EcZt<=~p^{ZA!SwlgE3%&~V9(#)S zh1fmaT&`rg{9nO4iBhL5dP^S0&16d0kS3|4eOIi~EKn=Zn>unt=s=qVWB2v+={s?! zHO{Z3-;>`>S;HWPQl6F7uLi@~q07!lJ=>Q|MO$`F`JqmhhQI7Fx8o%h9D7+M5!rsY z`eB`CE4!xBx5eBIEiT{*RvG_yKs8T2ju`^Rv_ES9CS#7}=8q=bP;TlZ0Nmf5v|b&W zcNkZ_R3g}F{evZ#ttgkhq+t6m?}%a{f3}Alf=`(?nHM}6D$X_)xJmAr~iYc;pJHuDEI%MJz7i@MAC}Pa!M(R={6^(qs^u;kcHKrgWnxEXHEhB@8iPC=s z6i6gw>RqB6oOyjJVZ^5?l<)D)%b;j7$k=P*G2X|dMrf>et2EDpNG99jOv1L2vtm)2 zRg(5w%w~xNEhlyLrSii+aUfUNzAHN3HKMG-cCn||>E`~-o&iVK?z4DN9W>HG>GI=6 zifspva6$edx?BcB|LoADM*h7!#qx|zv_A?R4mpvAqvkX&^Gk?+QiN#xeSGKeu;$f) z({H;mKf0rhVL%f)UReG$syAZ2wGv(S1}X?n9ao(V7&$**Y|W@kzBBP=hGydxxF9dI zeZwacxwdd^Mgzcyb?r4T9o$loGq#R^=ubU1!0k8l8A5`Slqy^Te(Jg4D9_!rWHYg` z!KQaRNw36oK1YX#k0R-lgMFlrpWVn;QB4VgnY9n2Tj+{B%l0nUiW!h^}W6M;qw`pp3u$FV22l`pAtQ631vE+ zLbNH1`%Re}_Ub#4eIQ2ytojv8tWM<%Ha|H1O;zsGkwyhm%h=FN&ud`-w z2;i!kk$2I@^VviiG$|Q`ouJ+W@?0B`0eyV zXcVaCS?!}OBiTK3I^{&4rm6EE+m=bZ`krUTmigr%IKo-0?bh{!^RF}JtSnD=7RpDJ zp_f;>UE=EC@8r*@b}v{3y_Nt!JZ{j^PzCTb=^o;IKG(k4Z*@kZrbkQq&hytgPHwN~ zKeWBm+t4=YyUL|aSOWB`VPHN@KIG3QieJQAIPaedADE0RMo#=>zl7>Om<&n>95hWx<(y z+d{MEg08KZH&fMLCeEG8F{rP#;ER4+kIA=_EnYjR{tLHGEmHe6h}!{3XYhZ_FCd%d ziFpA*e(4g{vVz#*hF_44McAR3fVOKdPcWm-SYUgp;qqX*!X6HRO}{SJ_j#?S))#Zt zXK*xfSn}z()YiDPq`tu{ARS6_hj%QA6&6v@LOqad!U$6=eYVD^+8Uw!#|URAvdwkn zixENEfKCIn-Px0dku(dDG$k{>@)>~w;x%NWqV9gKId}E@_^y1i*-;Ou+o!T| zO8=Iir@mL-14y<|6fTv(uIx0uAxhBk5+G(GtN9&b_}>GndS1F+dqrAsG2u57gcPmwt>i#3pr$-< z+R&75z9Xi(>(KC!s(P~l)4&aHYM=hSBXDQ{L_#Gp>6p@Seo3$N>p?X9UM&Ev-H>W6 zNbZFK`@Xf6=zJfqX_~F&^xLrW_$~>;EsWEpcJPWx!j(7;v{^N80GEbALBcn<1JJhb zw>VbQB) zH^5}&0Om#>%-RK+{gxPbO1OTTXOxn{2>ey zt1uC`MmD#d0>{vTMl|b}6ncu2_*)|u+g1Pf4&$1AamEZXKlPN>L``u+=`Rxjem4jv zyz{&D^@knwhfjsqsj55Fbb}4eYCre}QC!VM1hZcRis->+^8bWLV00Mo7mM{_wKzHr z@$8gi>0F)slA!LAP?l{qAK2EGGeSOhKXakOYqA1+oG8vbtG(DLosr?z6G_aM{SBOfB@tk0++>b@sW1cW-WkO_Ev@}Z|7PJ%UMTYZjVxCUp zVpcT%mff+@q*}l}vc^(`ix`{08lVS5TwFA@q{MKUQIB()Dk(+Dmj4jhBVr;3#(xxe zOK~=yi3fO#0VN-$EMMJze$8U(`f4jlS0W@2?)YhC80w?;cZHHw*?jlu-{kzHb(opD z;NK;bqv4Cj<`wa&CsN*uxOIKK5G+`#fcv5HJ&@<@kLPm(uO3YK z>6NWYEUm<*`Y4;*%~RZZ=3hO|>Z@ne8#MY!chQY+luSwGNe+7p<9u7T^M)!)#-F`@ z{p>Z?hJPsQWwqbwCkk&ibl+_LMUkY=q-1$+pMNuN@n&agcHq2^knUqCQ(1Leh$wtj z{?kRtj?b^f*I$3-dn>c#`%3cFK!DeWkhee2zx{dfOl#wdf1CY@wP>08H!Z$rrp^x* z9{2AesCC~x{pz^THwAwg)N?MywO=7?;Ka3G7ta7I-}1Js-Kuf3cELL;wnhmms{GO=ZqhS{|zj3T_iD}wyR0BGC(Nisq{Me+K)9fcpU(KyD#i7^zJnJ zmbmb>by>X)x#jUgiw%Wk|3By7N=k1?Ir^fcp)&skU~dKPK3f-53{d>JaSXgk5ZTmJ z+>};+`|5LGn#!hb=%!xcrhfLOf#s&oyOm4LL52(C)c?nSCLa0EMgP}-ZekOe;~5Yk z6~7tQ`bn68lwxKQi#C}rCgBpjiu?cj&$HhZE%MY%T*Pb>!fjz9sV(-k3>c}!w$b*g z?Lfz~POrYm*Z<=`5BYQvO=rfWBvMK8g3Gw4m%|iqUA4R2{r~vS+$H~|cIuopEzWFp zp>q%RY~E+c*U}8DdPmmL1g4czRjr%KWEr&wj#&WoN);97hJfZo(o4$A$EFq6|2%h@#2hA_+a%qk?SiFy0TBFCZp30nRUf znwnA$Omt^NO0emToUbVP8NQbpsXXt(4y5@e44(WglaYqD-_E(5bHwWr>3iu@rs2Ey z#V&IWvXT58F)n`+eNa;(4x5eNpvzZ=Z{&_7Uzj?nsY7uFiUYg2y6%D%NI_ z1|1)t^43WxFGng7-KFyqmLuL@O?ubfYKIJ2+c>%UcLz)&$d>b3iWAb8LgTgEvP%V$ zl{{%+^_iy(OA)uGb6zQmURDLlY^O^GX?p0=hH0SlB^Ytii&x|{r~Vod)GmrPZ}i@J zOFr@D*87`mu<9;rvJxWK!X+HZkWeMD2w$w{pnZHBA@dk#GM5Vattrp1VMg2A z8(<0Zu;`{6=eF{!Fun7my6Ef-X>$fGI@fo1^v};vU&a8Jn|DpKy77yBFyW4!{*!0& zUxw-!WGbg8F@O~nFozk7$&+&mi>ct*!$xOS5`E zlVCk$TIe?ECTD(cmD8oXxGZYgq;7$Yf2T+?b^eUu4##>Y*OJA|KZM3j$Dkf0-`DuP z+rM&FG+LZTWssOI?fd!UBU0vRE9l%_ad=!+w2t*o+;1n4TH8tuDpk$Tx%oLp=f&B> z&p%JShlO12FNv*^KEiBN65g7UCO;2;-n)N{^Jo9jQA?wyWI5-+>0H#&K9D9%ibiWK zIALS40bKmn)15H-Bj512nE)MmM(q?x)=ZX3!~_^-*;}PK0mYq%<&pYb2sjc&lKglc zY0EcBLdD*L;^TUMJ>SbVc)0$@i_m5Q_$;Fp3E|^l1+U2;pE1!{n-E_>ntzUcm&B#K zagtGiVqC>Jj?fk&Gg&ekJ4*rM_WAV7U0*Z&p}jKS+PXZa9_Nz*@A z123l-ohEguUtf!<2zj<%DbUYw@*FtU8H`M3^OHM+?wgS#unl!FB^I(0-lxO16d2-K z+e)HepDTE_8Wpz}0j1iWTkk8(cnp85U`cxLiN>khR0iX(+Rj~J^kTr&*wU~9`tR@$ zr9CrKvhQrqs8!8A3^nRAM^}fq868#ho%mk+KmN1b*_7{fYlv2Z`!)-4wcoGo*tZ(3 z%q}Djf4{m{&}#DD=26Ov@7E31 z!ku$n%1h_-z$*Efk7Ck(P8ab#IkWP~A0K0g$}uc&AepwCYo(Oh3A(1v{0{et_C|Lb z3YKC%JhZYQ)hW?z%Q1tmp`{w{tLk z;cP8lNO;TDu|(>-8z1w`#ImN$E4Hz8pmeK68mQu3TkA4>)}lF9L0Mh*yU@sgMa_Z% zwjguGT1yT0j?;XK-Atp~U1zs^EtR7}1LJQsm?S1Eyrx@dym0NlrmHs-uloJqh1WU5 zTP6i}A3KO|kFa;1GN3koUUG(v0gmG>6J6mBSAXSNp~(o|>_Hwm9#bBWlWm98UR_9) zs{IQ86Fdx`p{y61Z#k*cE46x0kyiTRg;6i-`uXnP#Q51~30pjMVj#2)7v>Hv6R4Jy zscf$avzZzo74>&n^KxUhRE}D$r{pB1dvYC|_5yzCezaxIWjip0ORD`e&hY*&7EB-l zk)aT7G}G>ZtY=3@R)<~0<_Q=*Z?fpdBA(xm{U=+$Pfju*+>uawfDYFAOl!{Bh6UKW;}Y7D#n05Mrn#j242ffZ}u2v+oJ^-ud)>}N{+ ztrGOlFA5KgKi8aE7o{WJK}W2t`@_BD_QP90TmB?@rN?1Mdm(TYVurtttSRg$O?hcO z-#_WC=+GL8LlZMD|JmSN?PLLy%nT^_N5V|R-e4$fS7UuVJGGjzQUEZfanm_}WV*3% z;dIq(GGd6$g#a9XU8Uhg3$z~y4iSk<&oogWz-}_*-Uhng1ts4~KdlmW;qZPC(;TR_ z2j@Bv&~`&Zt%DkVU}n&mGt2&gxsH<#*Q^Jm%zQS)ClbtsQdT8@of}BF|x>q9Lx6#Culf;8Wzz)xir!r&1Vov-%RrQZZfRz zc>_nRElGqCLGz1{=T^WLKz8FTO7M?!@l(;)$CDOC#q49n)X0dHFlhw{Tx~J$MH$-5 zHG`OR>F6i<$dnw#UOF3wV6EBdTBh_1$B*RL(RIUwOVSL8h{83$+k5b)05oC|GKxin zk^Rm(6-NWilz&p6DyFl7QO;O6i&=P=!sugy>U*HeA&_Bg?mce*BUZSZ0`H)JZafAi zIl)r*7$h>Jmr^oY;p+(T*IUVjv3V!=ibjcrf3tX}k!4kE&?FTCHL&R5Aj{^Fmp1!fz8W9-euuIIe6eVwzC?d;IbkGNyxwnB6P8u?`+%@q)z4 z3XevpfE7bn#M%X3{iD1b84>3ewebu%N<_TztfDJ&A&5K^0A4F4#25pv#GoFb#kY}V z(|eG&3V>K5A_NP$u7pUXlvA7-FAgBFrPP=I!e{p2;n0+13(Pdcx&0cZ$W z*20PnXTTZOh!ELI-bsKC6L#hB=pjK+1GE>QI7$&Q$%L8lU?_~*X)2M}RNICC$mpUXYoqM!B9My+9$iGt zQ?9(h^KcQ-g@lSD=bFx@$?w4Huwtgxz?~n}EQWhg0$0ag@)3mGxCYk;U%&PIQX|2f zX9@W9hX93LbL0yvT7q=j0csQ|nVy=FeDp1nE0eR8$k5(JU>KolSqY;HE2w?Fav5`A`p#?lXp$IN69dgNW?6wP;nXZ z9LWpBa@|dbuoh2@PBGtdczO9|n6VRKNFN;D2zEsoaikjegf~^rUDC|64SR|)U>0m! zp}Re7tZ|tAwvb#!c-MS;>?RE!Dq3w5|sng>(GK?PloEiZaA) zHpq1kGDb!u5s_+6j8r1G4?EP45so+_lat{rD(~ZTN@aG&@R1tMD4fnUI2FKCo+tI7+QmQGC?dXBAV5>cLX7`dFohI;YMij_m%&=%RGy$ z4dDbzsV%BNli=T);TI^pNmvYQt*nE}ZM+AWMW%>}@+z)EU9mjJYMD1S%8J=MniTMD z7F438?FWG8SfX9m;9hXJqB#YP1cZ@WW4L&yo$_F?^g$ks%~sfw;ng8+I1|A0e3+NS zzH$Fa31ym_M27Vd%wgnY-~(m+9b?cJ9-(#21K4QHW>YI!elv~lH5sqiqf(2 zSFOU$?7@sCynxKne-BQ3FQcTqK?>w9vwkRw^C~=lSwfCdO0hm-)&ZqNM6>K!laFw7 zC!_(R3~UdKIh6J`>mO-qoghP%tIrQj0WPM(_r!aT5z$_2&7=oh$-Y6|p=r?{tq~OP zJQ;>Rh)z^daU-`3$^@mAH=X;+XH@2y;MRObwhu4AKpuie@(?OP~(zq&t2Iq+cY8 zd7a)Vkhh#5C2eYceW)_i3)s|n22DUMU40y^MZz1F941ibhqulDGh>JXb!inx%t(iUC6LkcW~s3i&1L%MYf> zkUw+NEm)OLIlTInxM`}$*LCm&5v*uiD)_G2l@J43kIW0~!LFi=UMZekLHw9LKfMSA zIRn$k7j5rG5>kn_eyK=j;MG0&f~J>(ovIGCGWE`*t%tl#Wrz=fwWPhbIiU+&-

@94v}rjQEngM8ZoE^Rh}6g&?MlQR8PLG0~$u< zooDfKkx`ZRD=cjhR{<73bL~+GkAIpMocnER7ZFh zULo9x9Lq^YkAqk$vs=66#+3yv3$zwox#H#O{#gBn)x!We{ah9?=+MP!X}yG%2V3xYx~ZL`9o55TDEZgU12UR78~4bB#3O(KSY~axFEm@Y8*2<+b3V z_Pti!miuxl9htgGl*-+(qtaOLmZK_%h-hNl9b@JyTKmcG_?oR(f9VqM-h<1Z3pbkt zH2UOSoC0`Z5gTFW)9><5P`Ra?dOz8eXC=cO?{m%UO-NUU*HNy-Sn^z5%^Y)8Uy_H% zow1hkRxcpH8^$3I8q<4!;%C@$+$z(VFDgVme< zqqT)v?SKO7Vi~($LZS_DmcKjB@JfT0sO6OjO!A+V^mP7!t72}*Utg*;{SC$@ChFJn zSXLf4Tx^zS*T<|TW}L7#rW?=#wOcNPS?c)MWdpsxvXxymdhUuG-Ct9|s@#e*mXEi& z>=e3;l@N(cLUXfb@N_NI=vRq0DzFNMX`*Zd^0N88^?f)-7GB*#YMJ9bzW?4Z8>ZU) z0-yPFo3b@qaV+f3tHb!6rRGzuXI}q0zx#aP-1|4Le_z;J5N3%r!zfwvtbcb(m#`O~ z*>%_k6WW+4H4NvU9j(C};lFm4;u?CJm*VlFUIsyh8lyi6@AAe}4tS4wp{a%G*Z1&k z%?1k;Zhrj*gA8r(87~`PF-*rvZ$PbVTUip_$VLXM0hjpnSIHTWb^F7RoH1%2tcaI$ zNYC~c{KOu2SYvO_9$#bQ4D~Y&?{#zOyOsz9bng@nt}K-8-L#4fCeel5W}CU8mr2~L zF}is3?Z(VivkL!A)5&AcF(}3CjsF7JR-H5nkT?hTXTnqr*twB+Of9p92GpS+!PCbt z)wl@zZ%{W~HO!B4azT2|YjRM#jc8soY-wH}F(*}(o2|tJ@OwKlu^9*qJ7Wy-k7_-) zomYu9!#9bwCnG}cnbfdMSWhSFrQz-|h3t=5>W^ioTZ~SUuTdWCz*3Lb4#~C1lxpTV)SuAm)%Ev#4kLgL2-*37 z1^16UpxV;yC#PPs?-iC_%DX3@6VxDEI~hJ>A*R8SCy9A4EPmzJTOiYDC@}+W{5~~^ z3JR1?wx94*A+N!TR@vO|wb$4iH#pUlgXS_nMs z?0D`^Mpn+gWlfeZDWL{g>W%0%CvLi3hI{ZB-tpVX9K>5s@RP^+Zov@1j-$?}`gQ!U z!EZVc!_e&5f0|{ob3OmR09sV%30p|fGo!`b9@9fEUawLgk*gU~C$-4XLp)hvw3?QMXJz>ArgOmt z@2yjh@ZtRl=&H(5gRIw;f4oV4w!h)AYKO(v?GwnlNi#o8dP7pg4VSdNdHkgSi_v{m z{(1Vn_MKa~p%fq?EHzUm|K8nt3q&>x%09J5kRJXcE2_q@(QsV7F~(R74hUv&4K5b6 zC$RNylYutjoV9$%=+FYMIV5D|cAJA+$k`|tfKWj|-UoHO0gUCN#6_$S6>Kb$ug9Hk zS%dFrGOLydnIgw8WUaEYfmPOEj2P*mvR5DsSUsU@50vN=;}*G;p5HXF0}k2+TC|e{ zxJk4HfLMcgm_Y^C8UX`j@K_)SBrW?a>EWI=*~5}T13pbFPJ|`Iqtt=oWThcI^(fIy zA=0@p#jQ!vy?q&1oIE;x=Q89Jxay@n;M*EpJ%M^6LIres*;Sat3iPd9^l+X&J?v-R%}tS$pMD?!6GSSb*(5n@UZKC^YBgNVR7lo6 z0E&<)CZJ1?IdQ*AODrQeqlhkm$DI_)H;DX(~1pz{`Qx+my*yO(iHnIk zGn0Ip7a5DjW;$m-!_taPjp%?&p)ggCDfB3r1Xyq?Y`2C8JZp1u zKiJFHPK&vD)P0L;aS9zi|0NTcq-pb4KQHu;R-cwK3{x>p^Zi3xjJCagk@8&^g#BR4 zaz>?7Ki*p_c+hGWlGu0kZ*;hoal0CJ%Ix%qw#%%eHuvK*P)xGAwAd-|$ct0%Vm-@( zCIWqv-TA~JCve?3N!V|ZCbCcBF<{jr>cpbk2FVr5Ta!pmEFnv|w1>BIktlMf9*B}& zYOg4@Rega_k$q~|+mz$=K zDJ30(Z4FT-@pRmOXiM~kN1}dgplEPFV3QC|yoF$sV)D!UDQ5F@HrBF6_`iGu zdf89gl%iF7 z@Cpl1vrd4y5+*049*Fk&(x85W+(I8WY%r^f3f9-TG=p(;rFGMzuq2=8AXUx#%Hi9@ z=k>~idk}ZlC!R}7b!vkYE&{TjS^^RIHRL;7((rS>7N4BrscL$vjpyl;xVrlX#e;j6@fdf)v?6(OlE}wyetY`d)?9CC9f&XhfID0ZdV4#6Oa-1u@(O^-4TljSQJ=TtG4KooaG)S zM)P@*wVF{}UR01&P-V_{Q1D)s zdD#G7L7mDA&-UHn%Qzylca(6Neg1CHC^aa|3FJ)`z7YojR@18L#T5s+YT4YtL7G21 zqtPI*Gqzc)0m=!O(SEc!pnwA3T-UYIynkh|aN?zCbSd(m5{QPEj?P-6uec$2HGN16 z97r(}Q<3m*Q#jawiwtHiwn1H~vh@)DH5^oHG0VF;>kK~o4AC}IC{wqdYK#Yo?@%pS zJh|aK-boqZU$eXiyXMz`@5Uh>)r94}Tb`?g%|V*;BIxuE$b-?{P2h*P-mgc~e1w3H zpoMnUHZC8j*Xn_-ozTKE$qqk@pc#`NX>q1X{Rn5aIgSqN|cyk{&l;R<4FX^GgY-e`5dc&Om;x(@xLq1&s27thbT>x7th;^Pz0BjNP41iKF~m}=WnmaxxK z!{7Mw4@P=xPPN@)$Tu|_*N2g-dooaGz$e*uY&}(-=Nd*>OeRwQf6;ZG|7`wY-%k)D zB7%s$_ZF*0wL$Eeg{m#~j8)Z@Aokvr)~1TmqS~s)EZW*sTeXU!sA{!3esX!;*Y))N z56&m&`8|*0`}w@zt6cew3Ds*ocKNKs4{+t;U3Wk`h-~iCDs&D{73G0Y-B-!($h53R z<7!kMhhS5K_2trIH|AIIz!+pLa9@C#*&7Al+wUA^P^|uOYX@Uw z2h_LggbMZmh$-WGAV!<2AbG^9qX!gCH8ZcCBI%`;b3^}}aBzPtb~s<`>Y2b?MKN73 zN?a}C$pzBOIo-Lkf3r5@Re<#T^Rrxnf;Vzx^!b#sruVfm_7S?jH8&h;z2FQXuDKprRaU-^TF*r(?*~O_W@aDLMcU5{Fj)+11uH_;{2-$k zZ%M1lsad_g3x<^(mCp3^R_6>Abla4gH39JFWQs+9%ms<5bI!^kWOHd$CYzRpw}Q7! zYOGPZZ2?T)Ky{;i`Ju6T}!w|R`>f#Tr~e+0^6lEETa)r zEo;$PLJ#cr{u9hf^_V^XILqoOj@WE@(61bCC8kQKRFTxv{@XDxAqj5=OZj3W72anj z+=DSThkwMTwvzF@js_2l+;%uHA0Z$V;%hBuYfa88OM*)Wugd;CHq&g#YVo7@*d`gr z6`WLE>zQpwy6HbdS;@tfQZW~m55H*Kpa@2sJo`8obF^`u-VOe{>7u>1IiEZ^NaIiX^SH-z);A(l1@5bD^Gj4dCWO@Adt;`I)Tr4uaapE>=sFM5+ zQ-9;F?|ldFS=r@E)sF%>c1`xWhg@ZeXw6X9S)=zxy_Xy3MZ7gRHXZQKEQ)4bo(fRk z!VE5@aA1GgcB|i6A6Hy3Ueejr(Z1~E?A9$9v-0C(RicIL;(y%m^1Dcd%gw0tP0zW& zE)Veu54!&m6_t?KxSEj6<$@;(_|^qj;GqUjC=Vu#Bz68`;*yYhHd^%ltT~9|!BQAK+l~uHgGWG)97F!ptFo zMaQ#QW2|a#EjQH9-Y&aa<$a{U+%&;YX^R-xwzO56V4eHOZYp7RmkeF7pKeQs~hx?JY}Io(8nYTA8Bxr#{^ z&HW*Pb@;hK&R#ds@x6N*SOpD&w0bZsKvPZ8VwKsI#iALApt_Nj9r@kb;%x8kZ1e2E zFS*X!p3auVGv%?_sogHzZ9$Ey!H@pe|J)$*VBvF1gX=?8bjgUnlV5haCc4!4B4Pf4T z>E}^<6g=@Wq}2Ug(E;|7=%+D1OmEhg`qPmA^FJ5wZRKLew!WZS_UOUgUk2Bqullkl zCW5hu`8RFZ^LIq1N3!RpLqhcroVGT$94oimLjTYIoE*BWhhtsdIpB-E zzhVFDe?Ckz{CavC&PLw>IBkM1erEsr|NPIKB(A$2U@_9F_W$4iJV_E+A&Gt^iP1^o zoRJb@k&d7>%}%HuZ=$3y(Ix6>X6dZFx7^>Or*ii)fq4Xxo)&yRXsq^k@gp7>w94RE>na z7=yQu>C}lKgvU7DigC_~xpX(iWg{?q6d(=rNw0u_N6PUUx&i)nfk~ z5dX-!Fkbvd(Lp!G?)uz{eeAflf8y_#toYUQcwsEubYruXS)*<|;t+7<-$6J+_DaC7 zSUXNnpuKIX(zV{uwaAd{Vj$ ziA@TlGn;R)Jk3t{H7_@@A8G%~mm^N2E6%a_*Y4^`@P+WGm0h-XU#`7~MPygyrMMJ( z?7qFzy9xW9%@^mm_4CKiuocx0slSdmRUUp;XC1bk@3^_I^|bEu;)gCbSKO!K|GH~k z`uFObpjD55ZjrxSlZ!>HChR2QGd}ay|JuLD(JaY$EhpydFI7zXiv*=o22o@;=lbtH z(CV0MSa0ZdGW`orWh}R2HoxZ?U-EAbU%<*XKsJ>@R{X3y0u?SzkS|T3t$ck-pJ}=q ztG$2K+7~hUdGR{p@Yc(0&LY{y7xC&JDN2Cr(XG2<9$&X;6#1q&Xk!>6n<1nSI(X~a zz?Drpx%;|-;#aH?L!uZCk(v;ce`BL5yg*5~OFBf#rj>xI0KD`+b2<}y ziH73U*t6=P61q$DYH=5TLCebo4E%FQ+V7T)$(pf;s{;exYsn(9HeOL%e#4^|=NX>A z|Nf=Fm$@;5VkoFYoB}y{3V37s!NPn7xlg=1zJ;vSmW?!p2xMTN<&!)GUJ^U2i8Z2& zKb$ZHq{0v2CIY^?=0G0-RG|?I4w-CX4$;*qv>qgNh(;JcNhwdg{G8t{RTEo9;|PPS zIRrLPK;9tKX3<+)-eox6%f9RwVWQiEq#7e)9g>BJ?W9cs8~GA!o|!Sd4r_cs>0v}r zNc|(ca!aEUHkk@@fRu%Lyikha)ux4f64k<_*jNG2fN*xLLqSNr4JbzCLZh>JgQS27 zFz`JtH90TgGy(<7w-f3Ei+n#9cCk0Xi=M(d zg@Sd!pW|LxoX3v~O4n?=Ou~oEbavd}I&ATO z?aZH!)xVFvd|fWyS$*yLqt*ol%yQ1eJ>+)lGkRK(fCv|}0+7SgXrz3#q|VQ@*$&H% zJCS@_U%9$#{wr$C^E6Llkr?8JHW2(lG<`8lZB#aRv}#flk(_&V8>94u&1NZrdPOCJ z%yzXTe06~D4r`W0-`JRdFj`s4RLNAqSj^G5=m8X5&lsjV@7=)7sN?#^dVrF#XWXfl z++`KOJeI-W3`kL_SClyaFaKr2arJ^;rq2s+HfqC_2loHfG70y)OrOpPe?PSKAF{&2 zYqbi$47W0E*H-T-q6EOdvp5=!qy&sgkafQ8PU}|%rKJ?cEJd3vpyQ}3lIX(Qb=hct zc}klVS>w->5Z`+SjKZsAE3yKUEROZH0DVHecu$}KGXI;`rhQ6V2S)Mu>Sb~maFk=i zk;0aSsfS%Kqi}0|7W3CP8Smr;oog4*NcuwC&@oMNW@ctlWaDO^KqWz4fzXiEgmVh3 zbs_*+6Pq|*FP%ZE=Aj)BQdOOT-fpXk`=ZkJ1;FDiU@eHUk&(;YWaEyq3(;tVR1Kv% zt8Vd~b7f1G|08#w!jP*a)`I)d8Q#m~=I=RZ;~r5hzyV#KS7y2?DM z%>Y4*VtYf=5JqFBqCwfM%J4U1ymc-zPyME3cAqHn<$q#UZV?Zm%NN+pO2a=ES(;4j zal|`DH7NcjXeYiCfr|eW>$(@yEx_IQ{gje;^(m0kUp0A~DlRQ<)ZCv^9 zxfNLp@3=Y?q&TA7mFb?dCSdgh+&wtx=WYN_8Rm7hx=ty-Z>G-BcQecNu%*qdnyG-@ zT3~UiLTQNa#&uE}qWW)+{Pj{IJe5Ix%pv*GsT2SkVZzc=ZuPjn4wCS6K%_XG@ue@O zB!rxZke8luRP?-)R@4fs+hf-4+3(wbRLm@9GG_Sw$z)y&j3X(cP`)&s-G*O9RArE= z1h`Zu>LN*d__9K&<#mwa5U@eO3M-rQ$h-9`N0yV(n9b&f{0*0&I~Q< zj!#(Pt}ZIrdSf$_Hl{%Yu)JdQRWe0^qR))rzPT2;Cv|sBQqZ>WRAs){DT>U){+!zk znVnCpYzO}F($z)cc8q%o8l6*Qb{GDABaYt-iSAwxwfHSrxHnzH-WJt;%OC$4^xson zytB5FhQ*TyxsI_`n=!~$W4L!jLeo!;)Y~2DZ7xrrc&v$A|NhIu@5vlNha=7mr zp);E~0PQ^3Ou6|FtYi7{m8a>%T`9xXr}{?Nfc8VV>IjxAus=|5&w3tmc||YNH1)sn zf*DsJiyVFQ_bb;$XTgc|@((`lyWZ^zzIXk4T~AqFgkDW(xUXvE6eHw}`}ygPI)#H1 zBU^bPwm=hY^aV#|EYvYwJ7^J)!9gwFRq zjyab)pjC}-DRgj-K~@>Li16JMZ8_}>V&F;SJyMn0nB2Ubrw%Fwq$o-EtuG&0nGJDs z^?rCo8U+*4oA3QS$ZbCi!G?i~$d z8AD39=3kx!fWO<=$GcKqQF&_jlNemzJw#H9mCbNSAZJT5_&7@*!@=>I+J|KJsxT6m zqhCHnVj%zc1f+3NdNWNsWp@eHyzbTHrgw8i6j+7WL?sQA2SQDQs_`Ey_W)||-g|Ej zdGZ5`-_2U5@h1h0bI^cXQ?T*HjCOf6K5O9@)1}`1mt_PTbI6FwRbif(w`Dezmvw@N z@QFz`h@Em-;Nhr1g?MlgC2~DS{>i87}_{rCIln6beg&GGeZzVw=7Lrp8!*zC&8YF>^jl&- zdR@GbHnKEEy_aUkG|Z+l1WhftL^e7s8i@}8Ge!W5L4Xko@aQAjcngiUq^6t*BhXou zPtYv7MaD!X9a~(C^Z0?5u(<7*-0;}7LQ5+wYq-2rdJ7s9LPvjfeoa~%TIO<=7{^-S zQO1FD)|4|;KysUIf%iW~Vc`!xW~|kn&P~#6d`uIMI{<>`rP`z4m_*FaJ41>pl3c)J zesxS(T8*pgr_*R_HzI&nnbVDpBVJrWzJ97>AMIUg9vjc`;>Qw~MEWHF@8jVd_#KOX zm8^(9x$7fxrb|OFRovoFnEj;Z3?vAGicCH?i7f3v?AOo0_d)c!nGe+=tW|5CtJwk2AmVwF{tT>H>MiidZ@&Up){GUNxC6Hre# z;4VJ?M(u}~riR$pM$K$e%_ys zrL09mA3TZh2MT0X*o4E>Mv^+9;6jU0I>5U*3%5ChbA$@pcoR5)qFSzwNvz3dEGNY; z4FTg+3b&x>-3G0~_QG5#%2;~`AUp|-t(NmSSqfB_6a6t^eGjeMYG)K7{Xm;s&~ip9 zOL}CPSHP5Q60-3vNvRUW1*Bw3$z)ys*}8HdfQnhTrD9>_Txx4`I~m3sp7a`a9qTXb z_s{|G@;UZcg^95Ta9BCScAhRX+KzUyVMFxhFdQ;tEI&o8C2_4%O3L7dNJr;G==;^A zsp5W)MPKcYtc?7ef?D%eDvf*u&B`te9b*7@9mK~WBjDE!S@*DSJPS74nchgQAtl!0 zm6_ghy<*)JI=uOzjmD()AE5%xwh+lvv}X}gi7??W3M*tz+nuzM<}I2kAlj~ffsG5C z|9BTQ`=lwkq!l6TX=F`UNfLgORF#4p#$J{2nxvErvnmg(D{%-DCyy@^sI!^}w2`AR zO~*~x9y@7B?XAUz?TK*2PxPFw72^4HTPjR}nHw$1&Rq;ltFg7lNO9cT>lgGBO@AvB zhJT{WRCkj|Sx*}3qhk;8d%A?opEvB{JJv_JejC=D5d)bPXgk<6AFH0=rnNyxs z*6baYlhFX6O+k*x-INxKrH!Fx9TEhUUx^O znYQU!%KBqif&Y5zs|##){G4AUCAiIuKzdlV-W@Y|_9V!z#1R^Ofm(jl5!6&byvM9X zt@THjF!TT}^?l*Zbylnee(U8Hxy1YAG@Yvr@HI63d(CH^P^c{yUDot^AWmA3H~S^z zv%sp+VQJxR^Uy*ClMawm+k3@IoyNMSsz~sNW8JUtA~^Ni6?mvRti9x_DB?K9c`F1jY`r+T@KX*wvS?Veu?-`RyWw5Q_WdH#5GcF)s(4z1mY z=R+U7Tx_SvSJeGCo}Qgan;M%r8_h>so@cmHk$?k2nIbwjmCwO;yDfy{Xn)9TDeR0P zVfH2`poUVV%uoBTqB92I6J$+zs1WZ!@g{#$-@#Jv@m@3S=YW@J-cX6(v%Z|ZY}M{) zzItb9%O0`cKsc7MBT?<#7U7u-WAYK3g@JZNjPN$fzvah%UlO#Ze`sp;;JnW+epcr= zTV0Y~RPvGhGNd3f;C_Kkrr~jV15xdLRrLG*!dH0H;{Svz5Z#uw&QcUN&LwyY}m^+0Zc?(b=nQhF+G zQ!sz`!+A>=#`cByQ?p2CGuZP+P zSe1F=fA&t;hM>Li54<$&x|UQfzI{^Zg!N1Fq>7&@+kypxu;BBUF4`eJHYc+><$`B@lkfP%23ze}R=0V}G@mg=8}7E*N> z;Q`uIHo>&-j(9+UuXHfMiTGTt=|hV!w+Yoi3)*DrKv)Jn6*he>Y9Um7-Q?3|0d zR$BeVq|MpPzQ8~5q?F4pQwzvP=5__&@5y8r`a>`@9SL!EI2@j*tCF$oiI<6x9dQmw z?+_a595XJy19DFB!2J#ovzufW||{Cj_}lE=Ugd{LmcO>aL3v zh;DyhQ!oNxcibafl$p5k@>zk6YaA=owZ^nI9akdr%H-ejeW7b7KS|5WL&c{b-~mjC zy;pW20awb0(}srdP#y7#80#O@{Y+&oX)5_{W9YDv{^Yx_hBbUECV}cmA<)9Aw3hPFW5t>HWGw$DwpHAmrk$Iud7Tew1B8C0lJpEIBgjR-s9ZDb2iwBZVa>l=0um&(8)CJ_D<@<*@ z&P>h?LtfRZ9!3Z`Nv!8u1LVvIE$<+H^1!AnM`N;NNY!M}4at6Pb}q=p7h><*XK?$m)x!HbzV8@lNS#))^(=LLbuhODUeo^1hOe zbubV#E9;7$B7dY=6dEKlK_kFU0*Csza1X26|3LDnA`L%!tv790`sfq)k#7HFrO-;Pn}VrZ$SnqS?3swvLdYx8tKiomjbeajS-}$9T4FX zQ3G>QRL#k3Gk9s4+_X}o@xGun5$qnX&|EHGx}KS$KTw@1#JbJFo_*@&rCf_pe4Qnt zhz*e{Ab%mJDu`lHDH;)}yEO{>?}1Rs^G8idbZ&7a3JB3%BW@@Gn6pQMEYc+bTuW66 z0KRQh3!_#~YOsXl5Id%BNLtA!wdW|mqX!zm+FI3I6W~xw3&0-QfRhT4l8E+PB#Q6J zBhF{J%dY?sT}KpKPA6i#qeW9iDV4%KGGMG((UWRSVb%`lqS|Z0+`!I((1WE6A zz9003OuhxEOWF5>9oMKRQocsZ}~);Ch(-b@v!7`!qwDQLq7=o563D z>XmNLn_vSR=^4em9FJ>0Oafj~F9zu@i8G`~bT(8!8;(c?W-J0!w(rhhD>IcpwWIJ& zKVugalQkATwLQB(Ex4yNv78sxp?8fGOuRSHDD@Po@1p_FC0awrK_$@C;#6DI|mUwCp~FU@gv& z0X1;h9pJh7az=6O^wL@~T^^=k%Y^7#_6mJ9;HG>Q+LBNDmHfHs%=9h@S4FJS6- zJUmkqjO*akW>3@e;w>Bj^1h`CQ>dtqSI0iu0q(c71R2%njoEKS!Fb(*laRTuz}PNH z(HI#YKrQSgmIQ(% zXD&ndmXo>?3b$^d|Cqa-CMIrqNu0s6C5#(jjKURrqWbr0ntg&}f=Citm3=Lty^o=kItAbyKby&Nal#srz{`niwpyeq z%H6^!)>8>ooQ;+ivq0E&twlyYyDDVn6D`|Bmv=USF1Xvv@J7A=SfKF8Y9;w(KhbBr z96j`^Miby~sBt-%<@_oT z>RSbl)}hL{7^cZ$|1^`MhsqLnL;Nd^5F|gm&cn0tXkHW};j3i%Bx|QEy@!qK!cnW- zjTE^=+v4D)q+CsMO#93lD|`Eovj<~t8Cy~W)x|Iu{*09J@3&xn{LoIfkjUzK-I`61 zT=Z%=l*PK1t)<~bRpX=~7qfdRM!kIcDtO1X=G>x$|#dBJEUW{MQev@ixn@3l00 z_;4-l8uPiq+{Job97;(@^|HsfAFAuM#DK^7BXhP#51+i4x3Mrzn6Hp$<|NTrQ^f8K zB_!=XUJ|%NQ8s}+vjrN5i6@FY>#~D>820bt;rCnfNRf zRf&KW07O54#?h|tyd^0O<``AEIw-Wt;ASzw9h>E1(p`RC<*C=qZD1oWgJjq1%U2^n2JDJH z>P7eL1ePOGE()(>p74gt+G!XiZcl&|fc`wqQR`W#tCecgj@JnC>ra1fe>yjI?9zGQLH zJeqr4Z}Xf;wP!9_u&9ymr=!CTO_?R7oh0O|RX9Ic*QeS0aCFI=UU#~9;SD~QXX!tr zP)S^t-HcL|=8cHjwUY_+Y*Lnbl{J#(Uuqw&Oe{3>vv()&o8Vjao5IinCSM0KL4PnX0m_rP{lhF z2J)MKpe#r5v2Q69~<)S&!QK9k9z#^uSr(lm(Z4JrcI< zAl1>W_R|bTe5$Ir6|SS`2^9MBDLQZ|HNjPX;!&IZ-&iR7Oqz^wC96K!4SNn~diKK6K zMX3mx#o-d-(#yKvd;p)`)$E$PU_NKtea255(O+?L{(=Z9Ghlu|$ zziZT`&6)m#O-MceDsK5rYOu*QIs70_%1LLPiTpB8SC-C{&#+*2j$v5l%{OnGhpy4s zorrNZIV^FCsB>Y&pauC~dZes`oy3)J>CXp(ZX`*sa1Nh5sap@^0~!o}JOFJX|C+V$ zFp=c%tM8Pz`OaaW??@tSN0N`uaxmO#-?3_RAvOUx83Jmkr=hk;(Y$VT$b>PmwEI5y8P*iv*8H1gQ)RgUbc>d9M5t;9X zpTF}z>(KGSHPQ+${3^KkclIK^Kilq?6K8KoD6P}$#-YPKUoHG)G8%yDhzY=L$`J&1jtUrA_c?^FlP;dJb(5H^t#ioeE7aN9$jlGxfon`4^f6&zXh8 z&7-1Gle3ayFtZrAMIzoJrO+Z{&LU^dXc#F`@m+cEVOwwXEWJ(5o9aX6@|RTW4i*k%PF)C1z-FcX{Us{Sl#)&PILW_ zvFPt;rH@#=*K>%`ecYguAPl*PLFEN03q76U6=3oG-=9ZD!x)Bp0)_nbkBHc>`oGw)E}xN%Prt-F zp<@XXvi{zMkOU|n8(P3g>)Mq zO%NpqhN7JieEx`FzjWqhv!$W=fN$R&Upq>5efFT>LjLBnCAV;OVHK~{*D@UE9d$F_ z(z2yNL%(J(+?=fJtGo7cA|~*P&N?(KN6krq3~Hm|IJOzXGR-LNN1RkLn>>S0%%eaqmMZM) z{U1JiPrnMbW@^%J9i3&Et>1i0xa))vI!O{aQN0EVzs;>3P%N3$<@Ty3y5zf9dn~w( zmU3f5<{(yEYA5_M^Hq8YYme-KV|bjdW5hj2n05Zm`l7%qn$jz=9Gw0ON!p5%3RkP+ z()DDM>3r6;SD5rmG~OM_M8xjYMc&*=29Tr8@CEmwEQ;(k6czr+upe8 z33~WT_kbe(z+NxqvozTs8{+#cIzMicMYmJu)}Z7me9(ZJ{o_x&D5?EyQt`8^mjp90 z`_3p54dwW)W2rE3!<;7{>;h+AXP1!|qeRK~QJ$&dVDKYMx z`IHH7cN%N7luKJUXqosgXVd_^sdZRvZm z@ZZN>z!d}znsYz;sMDrAWG@+R;q8caWese?*ZGQDn&-zkYHb_;@D(3>>zy9|X=+3D z>5u^3);aA!C-)2Ie_MwrcTT-?cEF~W3d#E7a`s=7uKsB*S7D$}_O#rjmeCk@aP?^9x+rL2V&nXNF0`zmAp zC^<^%UgTunGq5qMm^PPP)BYTh<5<8xN)C_m zV`1=7zv&zLZ$!hQF6CBzEiD1$5TI?aYvAa~G7<7zUvcYw?byP&a`SGyMG|Xj0utzS=`e9QDfPeZ%yQRt z{bRQj{X)4y|2euO^xX$X+r82Kip-v^$!kaXA7!(_O<9vuI>mRadqcsIKVQ~VC5~H4 zjOHM_!qjQv`R#s8kq2S_?v0(_xEMB*bFz@zy&e}Y7w2K-D9vbhAenu`IX?4|Q^GZt zu$0H)n^%$nBfn1i0_v_xoG(`XD<)_;WV+qk>d3polo07RTVzAhl`R?5m3>Y$rN3s1 zb73BN41V9^H-O4f7W4$it+oo~yVkvb{q7tyD89KCKr{b6+1{Y#)g2q?1J(e9wA(64 zCe~i+P0&kT(y3teUIpV#UF`xor`{VtcEO{bOa5r&>)wjJ!v`-uDF!_=tO!K47)zWg zUrO3i%?)6^9>>BIKR`}gs_|sSKJKcME4E@9DNl}l+tXPMQ=W2M5*LZ@k zC2(L(6!CZBskPoxTzrd}*tk+kj=|kleQC#hlHHp&X+y>VaO zh(-TIM+GuIrnT85ZI~n>Xm4G9@a$F1BymS)P9@I}PfyT2ig|IJoKnx1 zK`J?FlI^Yq)7QXfu{Uj^4%!l#f|#_-9|;78ff<(M*0T+SN$!L0v9F~0YjXOAQB$)~HKGZM&Dv%6zz=IgnARsB$Ck4z78kZ$M>u1mZbEESRDSCXt2?X7;`Ccp5-1dwos*b)hF zUDK#%Jfk}(s=Ko08B?LUJut8S^(133__U}$q1J?rBBOumgEUR0TS*|buvNOgR%sA) z5Bm4x2o^qJh<$O|K%Q}-hJpTGX+k?gBGjhMmB=Lp&V$lV*G}SU(QL99s|aNZ6+~$k zIJF%+JYwQeHL&^@`~)dEL8Ui6?1}U>CrJ7_JB*?}y@nRh9Ub~H+TWH^MFiFATqLau zAp*|m>Te}d8I@-C|1$D(SqnzM0#k0+%bD|dT*{=cpO;a~;xRZnReE;SYMwv%>i7-* zZ}qtfU6)5vtJ?>2Gh_nP1#F9zpqc2=IYO@2xNR!B?H>*$!JH?e8#h&I%+xyCV4$Is zm8W}RJPwbokiar1aGaqMh|GwteOG9xs2nvOCNKsrp(z`m|%Ms~I^d@c+! z(>@rI)gg{%f#3;KP*tO(S=ai?D-3A)s|$-)|5ipYa2UriFG3F$MtpKEoU#&gjiHvr zJTuWWV!nxv856Wc&%8DdJVhR6TLqa4@Hj_Tp?pj(j7Shxxwstd8Fa|98AQs05indCUxlS-6j29xl^uRgI_ee@-nva4F9NxC+$>9s&J#=-FSB>h(3rDDh5K#aN_7Xf z?7aGSc9Ah;5EpxlCD!XbhCsln+;L1W{f-=BK&%*!FnM#lE0avePt%y8PB%X%m`-=|L_m7SwI$8adx8hgxMa-Q3dx!qyM+gQ!=nvIRJf7<*q`K7)|hmHGLD zV)TbfoEQ|6s1@*4vMF2iouuKLQTxNgY(J*EiYRq5TQXe6nfy-aPmJLv?oEnjKr3|a zB9LNo-3j+Z7a23#N-#w18~;vAmo%Ywn9lbTI;I@W&40(dzgP;cRxmNhC8^l0WkaeN z&PVjur;FU`HGyv?H%5Ek&N3s|HzD!;h5oCwc4HC3#)rb+v301h)0;M})nRK|L-3ml z4h8tJk;z`7nVTh+b)xUu)djFPl$oaq-ymo|%GlxSvzA4})A0wY~Z`#73;Tblr zNCHD06fiWnM)F2GAW7F1%;~p!!;A({)a6S1(vx`eZ;fE4iL^&vayXm!nTC+3O&7_+|=^1b>nMy5rxW>(%2)=zWe4 zN9)auGA{D?8-*%ryFqOPnfqu8^JqlKj3z(8U`6M(C# zQSnY5&oit}*VBI{iT9h+SNA(B}jq$%IZwQ z3x1RGo`1eZVpcGYamPHxOTHWu|Hk>|)1NU2o z=z@d|Zdj2OqbOUnN6O|kb*|z}?*k<@WOHd-${g<(VFSkK`kfq?(0>wRF3E_quD{0Z z>5TY_qsnv}>V@x5#q|n!dI{PO2*c0j@cQe5=SXQRzN}KvCG}D-Uz%(ol2wQ;{&vAA z8$rO~)e(RG#Bad3Hv8GzH4=?1g!C(&3TzJR=awjOe?I#Fq9EN`#V_O!6Gf`tWLdzc zu03g%V4zIzlVv5(>bu!1e^Nd{w@_SxRyGBZ)ufz^r47XrbN9&X5@uiN>t?YMO8h=N zX`OT!2l(wh;#D_j%EN`n{BV`qmlk90_P}Pfn%vt5*{_fM=4|;XA5vFn-{PI%p&Fe5 z{C&yYn{B*NBh2zb*(CvK#ym>w5JJe|0Cn1eSWzNeNlZTG{2EPjdof1zPS)(| zY$e~c1q^l(1@)^w+Oj)*QP}DsQx*0WSTVF+k7#k9B&Hm{qI=cB?~h6{NfLHWGzQ*W z4Y|no+=gzERwcxFqU;HGJ(@@Mb8mo+Y>#g>tBEkQb5)0zvaQz@3HdvaPstl~K8Kr0 zsQzI+&Yc>NZ?es5F1aNX$$g@Ft1#qlsd|x;UC3>XY^3qV)$;TyWjB)ItF;=MeGEOB zw>1W1m7J0CD)-VJl1y^_en%1i2%V)$nd2}Iiy9kpRzl7xekjIEf66Z%W{vhjQ87o0 zmOC`kN^`0^oS}GZ&ey=Wx7lg&S2-18WFI?0%ac%=E3CF~OT^OJ>@#PQgug^b?N0lw zoAai$0JM5Ec^wY0ZCC{?r8h0htbPkWgsIAn^I{OFvsvaaR!t|*vRxO|FgN^=rZ`tD zD=`L>(+H;AwLP$R6KvOfq2dq*rLm%@$^?;a7!OrD=gBQ$r2&=@7LZqw+Mg4 zEov~$H7}uR&FUigOQ;sd>vR~dEp_fG52FsZrt+$|>@*-|6t>SO3Wh1K^ux8UB(DD8 zBwP_raTYar$Q5EPc2Yulo4|-irB@G#FQ8!!yn$WQbV+R{iF_{wwSI|6v*f*B$l7Ep zW*HaKlM?Iv5Nf)1H>J1*7jSEjscF@S zzBT^&Mw2^HJY2?BA{0dx6W*mrhE8d1yku~Br0G9zdpL$U$jM-Z!R}>Z5SmesqdtQ! z$tA~S!~9LeX;UEUlaMtfo+FPxD+=2)L)Saob+LUHgo2D~9r*woVpp3?-~eySF$FKi zb9)a%wpkMT7GZ%=xq&|r3ENT#oGVAF3?xtNd8a~0LZEzUQ5Htvos@pCmrM?WnI{G% zmKZO#?UhaO-aE1uh+zEq13Bcy{Vp#g(5vJRhRe=wb^9f+RZ((=I@P`N>0KJjeB4G3JAtH0EE#bLLMbqE;d zJ}h4mrOb;5q?9WsLt?Bn$ytiwJ`+uzdhu4z%RJRX>!O&QEU;g%=-w#3HN|&vu*26? zUXbsJY1wS)GL!YJ7H`2r?D907%S>D@5MAn2sIxKaRm_mCOJclX1UN^FI1C7VKr%S( zPYxg@Z7hlj9G(kHz{}AXT&9J)@-t$vybV!YWM=f{4}K1yg7T^=M4cg|IES4e|0<4~ zI8{2kU3(E+_j(v^Ly)p1Natd?ouXy->g4`vWc%FA$%N^$eO1MJq+rg355RLPrzJgUEq(w`vRYW&>$VH%_6b}@Ih;hvouqER@+ z1h%5gNI>{3`9+fR;PMgS4)y#~(^|ysQDn+$63Efi(DaKx`6y5+|J>~{EAQR>B`HO< zM{-d{n4$tcpLZ^9$8@0wC33Ej@ZUTETv0045={TqG{kf;nyzJiE8v^uHEYs}h7vU8 z^MNm=g9n1lJI--^*k)+h7K4a0j

qH?^MAID|a|kvqE#HhUTiQ5Y*hcGE+0$+i>M56Zfo~X=dfh%&chdm)HN` z`Q`pR=en+QzDHDM?4k&@T}JnlRYnw+@?CuWH@dOHY1iz{{L8azgZzZ+m?AHTTnYkLM*q4jX%mNAGRd*1*^PXse3wosvqZJ~zTBV>) zvlyG?dp(Pk9eVTD}=~BWk=dT z26BhwZrqkUV~`*z5-|X>AjslC(E@q8V&{by6CIkY4*ePCyQ*ipST1p4LA+`blxryD*>`nP&9Oo74zJ*KkVqRyoB^3P zIwWoH0j{>Byb)jdB zx=5_HwyY-Wh}Nv4Vp}Yop(W7@GA2Pz*ZJRZk+9mngxQac1*~5ae8lM-O0T z^9_c`Q{oc3UnG=fhM(jLX?zkEv9-5tSEv+K1MlH%k)b$WK1`w9j~8Jw#OD0$u2%&$U38Kmow(BC;MN^B2`bOEW*QYZ2*3q8dcq9HAxCGM7LhlpI#8;eK5 zI>f#Vr8dgy3V9JTU0kePDy7tdr$AIH1yAK;5bvHTYREdTA#xllt&8R3SGHtzD-L$^ z&z?WzYTt}hwWI>&Ny@@d&B`FY`v}uM+F4&X)EhQhJv*5vaM$`%-MfdL|f&2HqNgMHG>FLKB$M$O6-%t9Hupg@rmiIk1oW`V7Nd5fwT>Xst z5uMV*or}M%mQ!zQf}AZpE?Q}GRb+Gf96=KIEOd}U(%)Pj?acx@e_`!Y&(h7Y{(Vvv za)fh^@=B^*LXp^ttmldqr1)7y@a$3TefEznP(5e;+Nfy&S=rUG=2R9iEU0BiIl{gO@G%}bBovoEraR?T~^DP;Q`!`N1qUc@K}2P)>h4^a$3s?iY1 zaU3%VF?8ZYMBt~Nx0_P5XBUzpl1>#4GPeCyziep+G^qBgR!!-}G~e6ZdC3@SNoHGV zPkIUY8gykvzP0uGbl2}AtX(zl?7P4G`Rn}Qhu5#JX=@C9OVf!GKC}=^)~om^;Ph3f z&X8|UYo%rGR8v^0?SSBA1K!)_Q|}8}iCF`^XuaKL(-@!atlfAb_yDd6*xvlXve%FI zUW}8}*>NksBD<+jJ6XM7)3)ehVCG%T>{Auo5h?vHxnODE=e^+{On=HQ52zeSRL%i^ zS$!`2d0c^Y`MYf<$7eO*w9?1Pmz`-&Ka-CsS{EzY|NOo)w+p?vchF3y=E|okgTmU3 zJ2D>?#08&QZ0zuEC>9tRm0ta@?{i;X{Gw)cbS1^y`YxvLh9W;WLK=GRn5bgQG0cgD zpU`6UJoUWoD;IoTJLd_0>pApeu5fQauq?slckdjxY4y*!aRttiV?6zTy@!7TjQ)Y+ zK1_an8=d{LE(5GS<3-_DeDP5;CB4o6lfs)r!~J*u4W+I6dx{U2)^VL6he7=CXq!Fccl@Bm)XqWYjBKA7>+$)13~?l)V?SUm5BN{;W3faxoRlcSZswGGRP!GwhRY% z){WUcR4;UuGaNdX{PKA5T5J{O>=Aj3<{mc;!p6 zPpEs#<32H&z9?XJ$9a!zkR8G?qQWz`=0Bwi9=@1zkJGH)wtc@ne5f; z(|$B`-9CG?w~zV-{#L-FW9K5w|CP(#qXySMeRAllsvKMMgG`MQeKps4)Mvdq_sCG< zUoE2%jS4gVP$WxZe_93S>QGMqZFgj)n%}jbV|8Q^Y;L}wR5R6t<(=W;`IIAg zNr<6OCx$=g+2AIQ@2vAUCx z<8k7!Vq*t8FFo$`?YQlpHw8qs3ukUM-Z78Yu!MjwkfRZ(cuYcH#BSpobUU+|j!E ze*ISe<#Vl#V<&&?eVbGd?L5M#Z}hBOr9a`zlcPs2e_IOb|97M6jeYb)>!JRj-y+v8 zUn;cLDUA3fd&K77sv!v^K11iXq=LkP&3*z~BCg-R!~HHc@WS2hs^t1<9^hNw`iLb1 z!RHIJBAN@I5{&e=^AvE$^gvTGqs|XG^3B%h!v7n&xVm|$j)*zJN}wel;qm#QZ$kSk z-_JyvFL(>TUF>}O_j<5{MN!wwEz8(nkaSJ|(}-mKp4?B@sM+Tx!&@i(=G&~TWj~CjAD_Ox4X+%zSf72i zseQ93^QQz!Qz$R|Yrbn!K!^HVYR<=fjrnFU=YDYD#dnRY{-*{8Hy)^~xNf4@nzt0a zBU&tiENy=<7Mg0RqmK4n2x$6#Q-Yn7M3DB)X>zO0)#(jTdz=t>sM^SJVq}|7rQ2TJ z!mCc(MJX?O0d}UvhWN-?yEL_GYn|gt{J7%M~3U!nG*W?qyKnrHHd0&O-2`PV9PBN+K-KS@EH1A7lj$qP!=rBq|9KDplm0`+mOm?D}#W zr^ltTaOuI>jG&i9m&tnDB%tgXNJ9Kw2YZi%@a`TG^RPU{Yi+q-M>T$Q&ZgT8& zZI7(9k(u-*W!s-=?*~4$F-DEbZ0f+S`^{y`B=E()<1kAzX^UVI*z;L6$egMsSuY~3 z*}aY=mklm}jsjOMwoQEjh29BM?c5C;c0Zfec&9h{D6j78Xk($iy^A!X{~c*t!54{; z?4FVSE}ST!va@j{mH<8IPZx;5n1eSwZa@p@2nXzdWO&A{h6%#k* zI6#CR%d7ay_X7~Xvoaq8DH@Rh*2G?DJyP$ajjZ17#k0n}p-pY$Epa9lU|Aa^ZAVG}G@`TdmJ*+lFxl4`SdE{imHhAM&7+N>Wam zdflMV3FDj0v9_az{4WJ>Ppq~TAa{IlUi>t!8tNqpX%fs4+{@Xq8Y(7PG~oQ2Ua}Lz zrf}-r1O7VQ3>@8&rUKWs*$^8Mgmmjj{T;BDkOEAK&XgreR}vHd{!Z0m-Ze=z zy1522#e&7x+5}6=?-zj#v4f`CFngoKI`!hLc&#+BK$$U6tm#(&3TxxiU=ohB?M-Tr zmuQRczcbSv_6xQxCEIvD4P;tw__yUAyP(ugzSs!3^f?t5a$ST5{uPRVQxmHzdWyOe zb!<9habOHZ@8Q1x1Jey!E7rR>k1GnMpVk&iVn4vKaA&5|_?Tbm9bx6=3PbsgFwP(p^&REVJa$y?ON1VruVpC;&7Lz^C@< zswGb5!Jf~XI}SD)6&u8Xnhw?*tj}Acg&9ZBgAJB}^6WNbE=;ByM5OmzX+->#>QN=? z>5S*^f2%Hyd9f6vBH7KPb98eI2A*+D91xv24k;HVMFa0cu!_Dk3gVbL=%K4rkO38V zzlcGkgNazMx-KKkK2@=4Oo!Wbm6K=!BTM~iy{X&!YlD%Y2wuX0f0#Gw(wQbyrYdh| z@IDdzWDxZEK1ixa>|Z4v`dpW_W@t!fmii#e|7zWmX}e>}lq~|~o@f4R%o4;hh-@1J zF1}x?^eVB}w5Jfhl?LdryEV)tE>AL9^$S?((+)^a3noQ>oClhIyWD)c z>~>MP9b4rhGfga^odqOt9R?IL{)8-J2bBjc#c50hSBEJnqRfsFd<~Rg%FGBZ zkf>`r%wg&jje7x&iOb9b9c`)@Ajg!cTDQ;NwSHs?i`26-fZg}vX}7BuO(aX#vHTh8 z9N?`zi>Rbbk|Hz5)y6(|Y)x@MQDzb_2vp`yuu>DXIrnAJU@M=Fo2ejm7A{r0Lk`2x zrh$#=S%Xt_QbN|KI?&cX5f=g=|oB{w)0)&a-|P!9bn+py8JHQ-mPuDkTR#?kVXQNxp184g&J1JvMv zHL#4z{L~A3UANPP zJlxMa{It9C(?CoQq09j*{?Rg=nN~6>zZ$Qr%xqYV)-po%#jX@ytwYbtOg8+z#%#*T z#PF;F;b2jk2V6S)$$7Xb%Gt zM-bN@XyaGqdbX@CUYTq0TRMnjte%9hSn4nD>prI0cf%iz7q%+6qc;2V@4@ zS#-rx8?j+v$Fy>Q68pw#AAAOcHU%Dt21Cvsm+EFl#c*U67_rsUV&|DWd8TTP%|^(5 zO4qXogjAF2fV=rdJ*mdT3{nwo?T8`Fu*u3z> z6XEq1a;AXX3`m~$=iswYSEAS}vpkWc*FF2|WtuBMfy0!&A)-S-LkTvT-NI6e;Cq0_ z*Izgtc5F^cUC7Q`RO=XN(Wyq=dYDrA5=qZISJ~B`I+cchsJpy;Fa8k|pBmHmDv{y- zOsXJhzaSzuo%4Xf@H$RAOY9f5P|%o2%wmd|OkZ^&;!MDI_S!Bi+})&tG>VGr79Nk} zoQ`vqe6`hmmCKoNg=y?`dFdm}L_I3{m|0m_-=u)P?N)Nk6*%F9M)$<^?x_GVi|LKF zp^dCB0meEI!-S2M?|q1@b`nX^t*(zTV}b}$a(Z0}7%O$ZH$EV|F!;UBDQM)GDu&$l z0;}9ax8Pke*ZB&|YMh8Imw2n|+iqiC1D{s4Ss+&f<~v4WE^*Fv)L_q2CwrNyoTfF7 zV~9RXoYhkAOx%#E=E;*|K$annu&n3037|9!Z{X5 zi^g0gT$}*9Q<5#boam4rxa6Wylflr%0sUo!N9TugNj`H@0`rp{*7q65eEP0>GF)^I zuNIp!g7svZhd+;tiO6c^ccVF%vc(92oOb}##S-j!6csa zt6K-lIi~nB-C#A^_{AY+##*W_hbck8A^G#n}@ zGw#S(1DSD_&z(WEuNlFvaCS4hy&O}^N)+U~!5?q9f($x+BxZj%t(e#B+ZD&>G%-AfL}z@D#-Kj>j{ zSP*XhNvt-AjAa;1yO?6l(`d}Y%ZDHJPW@5JCn|%SiFdMud0jc=UUAT|`cujyO*6XP%AWkUF77=e>TZDrWFiM7zjx-`DrMR~h3- zYgneH+T>N&0kRt2V=$^>zk+tff+xWG zpl*FeY9i=Uy4_dHB6QkSOnusi^uR4IFJDke)2y889F9bRk}dm`V8U$1)A#An?io2E zWyW((mI45IF`R}#Ru6zQ(Rsf!3>{c5hG-*iJE}wJ^LOY-B8|?=3m0&QZAyajs|+qz zY5uxYt6lNdNK4L0XgcmP5NdP!VkKzB4It)EUlZN6P~TPO&GV2dPQcYRB$#R zt@gX(=7!yKOQIT+=6z$z56`T9bmWMH#cHy5|HU>&8N#X$){+IjSElv%@;mub&bRZN zwTGv@Z`cu2nHN`?)DJdCn^EVYJam>t-*4%(h$ZRnM1MZRto>X2!9cT7ncbN@c7F|Q z$Z=9X!#qq15PuM+Y%{dMG#W9r1J`#QChYw1qpPlg-?acz9EFVp+Q_9hTp9o}RL9B( z;{yp=_+{N2_2a9`8`!;}=F+&jim{Aga0rC7@zUX*O@x`XQKgd`ne|GjAfjXV(MlG4Ps5~)Jtv%d0szn8eaEe;#&N&%GYlDld%EcJUWD9?JwMY zdqQi|6fZT)Eg}d(aEUqZ0jHDFu5L3FuJ+MS0TsE!N@%bNn{h@lPeMgCe+DPiJc1tq zwf9q&{zc{!gDzXcEZr=%^-n4Mc;KLI)pvSM8cSEkFmb0{FdVwt*nP)KCk=_6vLrpw zatD(7LJari<-hSSRl$!eM$WZZ+TFYtH1x`Lt~{*j%L}bTjKE)Clg=rGOQ>f$zU|Oeq|=&L9k^ZY-dGGAg0}iO8H{zceRy^aRUmO5LLt=9q8iTUKBqO%BX5}%~DqV*|$XX>Y8U`c<#5Qc!J z4+x(?I-5haQ(G;;d^i>f?Ch{Emx|IOoVvzGEC7lJ_;brWO=4Iizw3?zme9?_opqEa zhpv!q@zdQ8te)CBltpx64~iS2XNEEM=$S!r0+lv|;ZbL{kDP_{fTb<;X0|iKnn)z5 zbJ(XRaBV7)F9o5`BaxvNmd@to!sg33i#2WES^pVlZ`J60@8bHJlBkvX+B*!XhPui; z?}qx?isFWb#up}Nw9|`Q-AU9gU>4{?k@FM?3E0u!Ck#ORgwv~mRJ8kgT zZOLe;q=TgBQx6s}*5k5|&%lUeZn}GOwdHE$h^Vdbx_gM2V)w$4<7*0MQ~RT{1vlnb-^E<0)i@GJp>lMx&aoBTSIp#Ay)=pNS zC5{PRb>?wEB9Enj*V^E-toF0krM{w$3eWZD?8s!j$^W>qPePL3yHD+EXfJD+t$+YVydJvKOU_-c~~V!?zG*rSu0AcEYpFi6Rpv1YaakaT20 zc%H!w;BZos&g~Eo*>gVcAWRX$VIiICL`o_MDW<*v0S_!WTZaM?PTID%IKxOU+^RdI zv&DCuYW|%drlxRDEzQnHUZiAaSjAjB-KEMcf<`Y%T7U~$4SH75D}*j-lc=L1D8b1! z;Vyc){7a)mG-a?Ioa?Lnvi?M+u!zi2_g&IAuXL-2JLvDFoD2J7xlGQF1FMM{itz)$ zS9yjI4Ym{2(${YH-iCOHMjci=@z&=LEU~p1M<794$^so#+$PB8ZT@k6R0LzExu@dD%NGYl~e|TF+7W!>X-Ut6-lsT=$BK17v zBaV)@?k({ov~|s;zQ=}FP55&Zx+|iUb>o&3a|;nT=|gz_DTGN@_7^Z`&l!H*_0Nr$nF(K^rsY0v*A2e*%R#(EE+S^j2UA!&T} z!Avnln?xB$AR-+kWDz9_Nxzs1((66D@LxLNq0}VZb@{9S@58Rj-&~fXI%!atN&tf6 zLty0Itjv9Zi1YjowOLLCPV0dMJE#Kov~fNHflrxAI_AtCJoTf%O!5WR*(j7GP!@Qk zEs)A@va`FH(UyTFIy?(d__ur zXG_awBFm5iMVHk$AeP!|KDWh8Gjg76pp;>M&6H!7dF9xowhIE!85q|IED*M7S8$6d z^sGU<$?zR85xx(8-jeUbjQ2lx5-}0})B=h}Ckl`{x9yiRgyn2>>;ITA=ME;z7$|mP z=bRtATaQA5X6YxgA6P6a2Vg2p`33!VkntN79+W{SN3s!tHW)+on_@}OxOk@w`?pY; zXp&Y$QI+{JTF4}&{r&=ZybIID)b&Y(xSTEK`ard}3du&u*(uRdcj39|9<5L(iGzFd zQtMaRAM>c>I42JD^$ZESAY_j_op<-Hj7`1|W?c37=bo~x*R($cnB=ElGzu_eg4NtW zzuH%^Cy>R9v8L(L%QHf`rNA--Ln}IZ#omzW%kjj5%_a1Pvt^UFSAwu`6) z{hT}oT@|@o#gF3-Fq~|EG}85Jv-FAv_qljJpw>0WKA#hloy5Gm(OT~O$0O4?htuvS z2UWt6k!&gii2;B;Fhl4F08^HiNk*#ggn?6N$(XtxKDer=NCb(Cvn;fQb8>{lB40kxunzEu7U zuPBZ{6}|`s9-HK8sl1c?;)E)wdmnVcGWAP&ZO+1u;+P5=UMWp=N3XhDD%ExUvxW&5 zyzln?S{Y)6V9`(tDJ2pK=Un6$+nksyW%Icu=!KVx1avbWjU35y7{Dk{de-09Mtgvz#B&w47*8=hE>O3J0^gXWMR4}D-7<~S-14) zf|t)#tzvf|GMK@GY+ye=8Eu)apK!Z%A69%pqkR@@pIo(f%lPU^) zC{Pv*hy-G0CjME9Lvu)?sRIjEiBA?Mu4R&?Q-SKYOVYU>hd#X6-s1?+ZO+Y(FT5U> zbN9XyUXmXEqLv`>%=e5z=%zu`rtq&}n}kZ3lr80NQakbbJKV9Crq_IIrg;lRQYPf>z_R>tH(FxBLOPl~zyl-eQt2Zn3(f zQ(mQ_EQDOgBo{ByxAcEGjwe~Yj`SwKR&-pRomd@}xsi#BgnH4WYA>hQ-cr==OqpKI z{gQKL+zmnLKEB*UT88-^_A3ylY+V8*qfb@Zg>GH0uX2JYogzrM+a^g{jeTJa`^zM6 zin`uaI`G-e+xP?N*r@|i&Si&JaxePR?|&Qjld==7x&=~2XD`pp`*P-Az%unn{8Ss+ z*Qji6-bpva=jZ@kBffx@oOB8xGk#TIb*2za(r%;wV{crmyAj@S3_MKI$OG_@Zn9&2 z()sZ|!20bh`~*#BBJz}dsLRx%*VyO3=p~?&p-WJV_nVlA&5u8~%nGN`p(@8dJJ>qg zj|~Jwi;o=E6!NiFifg+Q_upe!y8X`pJK5VDYPRM3f%%hRuRo@}TDBZN{9R?ut>pN{ z^ls`C*))6dDR>>Hx+A>WELMi}a_i*kY|^$uE#TV42E`L(jmk^bb!Kg~ z$Apo|*-ma|!KV}_Bw=2cspL~$azp#dM$_J6y8MFKWL#i6#RE|T2u@S3t~Rw-63D7W zOZvcv?@qs&{^{?G%ORIeMcQrWZNV&j>VN!v;=h(^fh+w-u-(OP!!);f;Ylg~NU(sy zos;Y9eg|ud=8oZnE6I8jk>;>cHS~{E$Z7q+>zQmPMgHq+DXYF}p7uc`eJ2x#%%9LCqBSJXA^?Oq3k z)Kyd;5rp}<**y`(CB<9k1=_J6Dl&>gyF=}|3g0ywO!ZLS$;M0;fR0tz)X-ce%M1iO zdjg``s#DTc8Q;ZfKfgVvpLT7!@ke(_q0`&+JcaIUg$oXuf^wnP3*C4fDWd{aBfTxv z6|u5ap?jlIVPhBfD(#x;3{J$az9=tiIQ%qsIAw3FEo^E#*Vp{rh~U}gycdEFj&AAc z_d%)mRdcTY7jpP%Rb>n`*ZC|d;_RsS+gak3U?wfnUZ|hwq1=z0MW$DwsVNXNf zKaC1sN^j^=eqUp=+xz+5`fSru#rqh>yEJR5XW5g_9`8-MEX_2e?-NgSzBfHtQES$5 zofp=aXS}b`OHY29rLWnXWT3HO+`0I-i zvA^rSU~HzX!|TVxt4?Xw*#!Pr4sSEZ%{ZmMQo}p!mlHcd*JD3SJQdnb64qGXt^GI{ zvFZ2zjqtfoQXfCHz-iPE>&fXai$A=TKljym_vz~oLhiNivL|;VRKKeQKYQ8oD(a1+ zX2iXy#uh=XUmj=3!p)018r;HW?qKuR$&Y{Mv_93eta}{$xBKzG(XhW!Cs!ig-mu<( zJ2>=2#7`-Wzu8&8erxT;%DR4yNLBuj;Gxz0`6J{E4SMt(`y0D#;tJ_tX62f&j66hJm#9RL910o2`o{L%gLWT|a?&7%_H`Q4;$ zp@1jXWn&+fRQL&snbn*-;xU=sYOQC@W&GzG zPEOrxe%?jfoV;x>5ix>&n9L&zD}Ni95FtZA%G+gxIwnJRO+rP-@Q8EM^J%BQ=*X35ox08D)R@AZ-be{RB z>YLK&ry)V5s{ft2bxvm8Cg+Y`D74PSP1t`pR6y?c_K{0ILb(<2PFr$N&Zw^(I7oQ7 zqzs)E|E89eqPAOCB|fTRec``(r7U5|5aFu83p}@-Bi6yAa+qMDG%xMQ}2{h9~O6XqN*R2aO{w;~?K9e}CegP__rin12$uI4Et~>aOKCjjyuo6Fuu6k6h87e0#)4Z=q_CFoyp!t@k`esO!M-tvY9C#f27- zmEN;+tCh8*^UKh1wWOU;vCByDQdt%Mw;|N%r?Nc?1-MtU`qwnRFZz3+X9bj z{@N-yE+|Q+VSl1u=!hyxIgi%=Iv^SL^Qi&(QHUM|vMQDB9AP0MI+iKXS^vwza;wIW zOcrXId4Ec|9(UfVGGT@lfdAU`dfM>g-!B5vR)%v*tizY{1T}~n1)nv3hu)B7N&Y>0 z_|vzan>h!Y0T7jhpE_T;V_Lkw%e|4kr61?-^8Wk#L+d=W{>vQzzYPZ@#ihd>SRjdU z4%C>Nh|Xm3sn`s_y}1m`Gz)4tJ|K9J%fuqfVMlESg^RgKIEQlh@$o_NUT!iWvm6;@ zGbFXbO(9N~3m)jcDf@dSDPLgTpNA6}Mj}5M)@yHA-uo;m@17P%-nw0OP{c~kd{#bK z`X;$9HXrNJs&M+Nki=mf>$;@KOedKQ%ypPmgL|$`n2dz=0~<>fkD5#wA=hjMMkgal zNZ&$4N}>|8zh6m8w=Avr;khx4|1T#jG{IF-=@wAIWi-!X*j{Pq_?>jm%rr;7YV0C$ zLRB~~>*`inu6a+*bkUrBRfhu}@ysr*Kv*&V*~^FGQBz8>eA!ieb}=p&jz;Hd>vzZq zydYHGT9o{9>wJNdteDH>sfTj~r~RLbJ@1}6Yc5dmvTA!~!)D~{LG7a0g3X@z3&VZX znrjgtMT5qnVOFL~>P6Gn=mWw$huLuJlFv3sLr|6_kYF& zzjM+Z!x%_7X#szTX{&YlwK~+d2KPSrnA#RdoZX!)-zvcGNlVnLTUBx3ed@lNUEoe> zf?24V16}JXYUWJqM4N9(q#V?^DQgp=8Kn)es?L`~22=Q>ZS4Xh3JMeIZ79QEXU(#q z{5J^@y#H+Kx8|*yE2V0OVg%acT%sG-bti^c3U{c+cd$2CDJxNqMNDfF~E8w=elV!ubhoxUbPcSRtz6rP%SbbHtuCRmHrKft+ zD~m@!ge2?es%x^eA%&D*-Y#k^5j`9}??JG;SvRok@waLHB(f(M5pxIe4X^(`18h;? z+X(MwU=v)e;=Mh288bIDvp&H2?|2MJz?}i|QnN4%J#%yxi4UPu@_h4ql2_R?IEqV6 zcgAc$F4=xP)otab5%}}biM}MYh}aTRvO={cI*RA$4Rr+_6!U=dl%=oR?4<-6v_w8^i0Lx2b+{N3OC2QADUmNgE&V3 z0EwC3LY~)L<@w?QYgnjoQ%2m6RWa4|!^wcP=WNv)0HPR}F_m0eU8HV zwKcWEI(h+gIVmlbaY451Ymqw4CGvu5A0#xg(dkng%5YR`v}Y_Zy7Kj6cZxyF?Aq^? z6RloKN{|+8W|-dQ&?6cX^svJE*(l?RslhgEs2Sua7C?h z)EO8sSkF^S+KDr{ZPr@e^!Ot?K(+@rX)bZW(Mj5b#t`l(TT1>AzZa)=(s|HaT7{6{ zA#wy;(iYqBse!)uMD@MFy>qGfni|4?(k7v50tv1TpGK~~d)M46($ZKW8r!iBrM{oa zwj5;g3|ct9RlJ|q97p~5T5oe|A6hH660CM|GZQ^0!#DA^U;T55v}=mf7wphdL;=M0 z6rYUEP1_s8sv-N{J;T%4H@)--3ErO$WYLk{LRLSOE?R$rJ)0|`2xBD|^=~J?b=aWHDUN~e%rAC8I&@=3Ab$|lKZEqdo}U)p zw{;yGVQ5B;#a!+(UNN9Sol(zHuC8zqJOW68ej-s%63+o8docGZRh(lGOLSO%BxE!P zRoW%yYz6k~mbe+?__)LSi=OA<8c*v_pcgc66ZfQp?oySk*mxEIPUl}D2u5&I9`jyb zc(Qm&is)5kXuz)xdANocC!oeD$dNU~1-AS3HG~R{f1I6ANJvn^!is~D4`)v155W3J zu;tf$)iY_{YiEY6kDmY{Ua+qsNqkEb)UzCvoS`L>3Y%r2o)O^3DX8ldPb2}trewaL zB5MNC1M#R7Hn@kA`HTd85O3-&03+L90*Pf z!1d{1cwQuEi7k-DKDB~H_yv10sK`(WR0B5UQdj0ILGT#`6~Z%w8y^KO)Wt5%AZrZV zMd{!rE)2hh&@jwRTSi7wP%AUA__cy8&P7=gn7vl;jEe|ZJ6|Mx)tic3AqYOgW={`7 z`dP7~M95}UwEJ9mGK7C{Ep416D;EPDXY-HKkx4oH%T_SV*#7!M;DQ%2Cns8k1M6X> zJzTq4IKnr`=2s#h2C2vq(#^1(o25I5!8OF$5Vv13phW;8CjiRZUC5AAy^i5i5C z&gcki7cAujpgxbgucU%kSosxS5sMsJ zXtcmYP8yN}8l}RYv6X@$X~nIs9}LozJkoTAjPV?3Tu#~rF4Ug{Jvj%=WZfL0!2Q|$ zVT64GFCvq}=R_441!QEht52>W`q%>QbYy*<*e21X%P^rZ1{4qAKh(=lT>^TuP=g$Z zD-}M-E$BI#G(m?EX(tzXG)UQOcKKVxBbvaa8Cdo*1#~>yXB6N-L3PnU?Tf4$4#bBA zU!=pPMgW63$c7nYC7b^``p}G(U|Hqw{xl z2?T=5t9;RqSn!8b7VAXBigN|J13PI3Ucxr+c9r6Tj^ zfJAR%Bp30F25}gMJmbPjLSPURcQ^+$oP!7=f&H<_M}USu{B}K6@ey2TJK^R>H=sWi z*~i23HL(!w6i7P@fij1V&maod_|Br+%ZZXQ_xYW$d@Kr*#Wve>MK*EZ4*~Ev4lK?N z)=NS9bD`Ch8IP1)kQ8WUJjk@mXRIEjM?(kXpi*gG4>V^7_+$CvW{~4KOi`-93i?J37IuAq zByxwY&;(@JjSRLQ9LskggWsH+%j>in@8Mr$!7J9G7GFSKbrk`CtE)zO0wsw3`14fS z?YY3Kk2nDl&ww4j?=I`9!r!`Ba$&%;Ub3uP*Vh{2sW1nU;xjDlG7l1-LdkQbK^Eho zl=}ip?0aIqEexs`cnaz|lkZIelFkdYLWyJ;M?gI!L0^f9>anW@A`!PvfV@^9D(H3+ z`c^OHu%W(JUmfbQM3m?nbcG;ARY5Fbv&A@|Hg<7-dCsS5A4`JZC_?WM4fPNWR#y+K za6^xhTrH_cMCdV3RyInc&1$)=YYCZcZc|zBQfVOAKv4C}LA6j*@Z1By(5NC`Mi8Y+ z)B%`95ZpSN+#l85&$)2R-7|s;w`5%&Bq7o6ndf+a4+OrubE&m4)*Bkh$*uujZGi6o zjbJPEhIim|T{*t^wSqy)xNVXDv1h=|Ke9D5C?=HuUyY9#Cn{tbZ%)Zvw!Ohpt9Zr< z%tP>RU`FKh^@SlD)`cHB5XfJcJObD^SafzcTwe*&tVeU`@ok8|MU0Y0Y^;Gi71 zeGQ?5ZY?Gvbm+DxW&x4}#0bk)BO`(L)d`}_o}Od%<8wkY(3K%M_vc^|jerL!XgnG< zN{d`A&Z4V#8~IT$`o)N%o4-#{CUU~X){8$KMWY%D^9P`toMPv;+zK)sPU^i)WmL`p z-srWsV#7C%dz69aiWY&faTknv*TR!J&l}Wupnkh}37h@95a~{g{T(EgAKA^UY5ZjE zie$l;aD&vNz^<}DG()qzzBD+nu1ix0U9;hq6F-^_%FfIf+KQJ3G6tzprx8}KnlC~ zA_$@SNBQ~*G&2yn$>B@09(d%TG`xneTuVG(r?SD%eGV?&Q0kj+lJcghFdwrL4$cqYGb?cvQClrafthR-jHMEqC(d=pS{ zDM^xP&9|6?cyYhbb-Zo8Qf z7brrgxejxh7aP5asf{kIf{#!@ly|mC`!GGdp0h82!jrY+#8RQQC*IY7sSA)`6DjWK zZP$(!&hNX4>H(#qaWN-HE_I3Fb4tD!xUJ0e@#x`m29iHRju}tiiX8E#=AZ^yvFse+ zQ~=tKXZR)%fsiM8HoQe#0(MU7?@J|witOuj!z(o@K;HBOC zfv6g47m`q+WvI%H6M)G1o{&R4BS4ZmXNqQz+lZyewYZOaZMjyrezbRu3krRkhE^cZ zeCtp;ILXM9mA@r9k7)5cLJkkm+|L%s__3r{T6MoWgK>E1nxjE-H6l~vkQnV&5=$l` z8A%kjjQ<5y%S1jXtSNs4@1sY?e|pav2GqWwkwyWHj;^5f;}dIlZaV{wGaJNch)qJ~ zp_=l%9^@Wi3YM!-qo#~9M@h(#t{m@b_$EsmJiE5uF7^zI_^vJ>%?m+R z3l^@v`JbY5|7ZGtp!hc1%xp6Zo9o=KY3|puG1ugNPnl~{Y3`D0V`FZ)3sLU3BqSen zG51Tk<(eq>B9$&gH(%fX;Qhn<@qWL~d7bk-xW%fBZLzt}t!^8O>Ry5F83Qq3|A=(r zcLm|EN!C7{E8KcPohtYWmq57VPQq1t77^hf_y#xNndTbPw%bQWP@#(%BGJ^3m>Oh8 zB)702i3qd%g(QA`!~`M0KKZ&5n}(sFTUWsuE*Bf|QqqgU(@ zQTPETk(8a!v?vN9^N!PvhSO$!Ytxh8%584Nc0IU+j~h~cSzOZkMYHgV0myN8U21T~ zb^OW|@zjwTXZjP*@HU(YquFZ~Xl+;Cnblh1nL0s>I*s54yxmF0Y`Tqyg5Dr&-%P&T z_kPHg{+xY_O3e%s-;_EmK{jLr&R#>FY3i@M5p{YyH0UJm?@(Ta_#GSftHTk{1qGb7 z-S2DX;a_hD(vDTU$n%Vcpf#`&*p9%YFd& zvQ*5RCtaF7RUu;f#0r|p8F{FZvRw(W{&hD>$k_oqUC&W)X_^6>XiYS>F2wv(^e@!h zd{uo|@rnVTR&=a&e;m9%KmF0Q(Yx=K?3tNu_m&IO4gPOtKAmX`S#+V;kz5-ZjMCOK z3myGA&H9V{bThM&%|^0V$&YrRoQHg3ZktMK)3wgk2+D<&`fu;7JA#vy42R1KjzbU? zcPC~nwjNxH1~0cCP6noz6eAc~28Fv@4)^tL^L-3>>82s6D0!|U06r#eI3~hnKg7<_ z7CmEfv1e8DPJKu5a9Y;&GZ}bE0SDI)Z1Xp`8bprO*?s%v;gqzNv+dFM=$?tTMMnIWJRwDW@&HIfYbVRrXs>|<@nn{0a>;{*M=ro*{Zf@ z9Y@FbhYbXLXE4Et+~Qrz;nl3hNdLY^sCWW>Y~QE&r^1H@G4WH|5cYEM*i=HK*L z$$nOy_3Nnm>D|KHubbBV8dzpS{Ba*~8NIwybGpAk3IJ*Lp{2}Vexg!wiX#I>OM z)u1TDmAO^d`%8mnXwpL`ytA{)nH>d{mLUia8fIka}x8W8%0u9y+ut% zE0;CipQ|V>+uf{qI01dTwU+YXo>$##GD|VAZ6oib4v<$0LgpvanGW41xn3~>D-s#y zm)DX@3lEs%9at(?Rj1RVTHBKsgp)wz@(ORw3f1z^!1}tfpbxcc4lx(1?nM5&P*a{4 z6LeU++06a0VUQ@7j*9rjRiLB(;V7)FLaIvU!;`8Qd{&B81Q)!CeqC+w#K!S=0in{` z%(;(C8-B-Yt9bXEY7<~CMm@O=i!|w-|4#n(q2JyB zK9JfIYfB+at!S(~EcdqCqqHYL^OWXBo}*H;kB34>v596sYxd>MMHTilV9UpTrM?ew zvLC5ccD~kBn*Mkz&{f|9+APBXr*2A|^DyD)&7m9MIbg^(V>LXX2e`sPXM+ z8{4f68(*5F`~zHGCAx0gVPKUDf6TY{XBA#I-zzIV0ZWT0b1ePx;YS_K8W~x*0vp?D zmyGY9fW4~V7F7GqiTgYyF|iAg7hSXY2sOLyvFPI>#xfDLgYmwe2$nydwXERzTD}AVrgXbc0c4VS!Y0JXX`3{dMP3S_rgVoX2@Rf=fC~BlavO2lcDRjyu zcf14_UGS=mz;c3?cwa}Ld!2UUb#+_%GvuRPbL;w!(6`}B@N3b$5U@5pM=<#W&dv&= zo}sO=JaX#nAW=LS2!<6V#SOMqJNVckT*fD{4=RaVE0LhlNQz)bo5glRH^3FPs-UpV zleIHLx3Rcrg}z{>kr&4b&XJVIP)&6x!+d|KlCqY5>1qt%$=p;1w}X>&;+{-p$DcSI z>{E*{lPSeGQR^Fv`Jl5HBG}N7p<3TGb-ttwp!t;HteOi^KdXi!G=e$P@ZDYuc1Ut8 zlqosn6+C5i+V>Af)S)hi_%=r_s?EB4x)xf3#e?AXG)OP(tpj;HDTq>Pls-N&;lu}+ z#7Y~Wul8!qZzWvREOUJF;%LjS`$_SoLo>8y6M7$%o9$Ud1fHi1$PtZw344|S^KE47 zs*l#qdLn)j(>xB5sabg zgfnyhCRjS|Aaxu|%wFSs&HJM%$P3@7ZD~0P6(akXV#$h)jxaXFZ zr8E_AF)XT>NvsV}JBjthCj6w5w^sCarm5C~U8N1V2jhk|_U2wtgm}KEcnoog*4bw= zoruo8qLoTjjY~mn9~1d}O>ylILJ=CPd6Bx_wmHAR4YWo*C!R+#YUT;nhygg3lC1}-RvnFuR z42Pw%q+OO8)i!yhSVXY1olK-s!Fu6|!Zh>oB`_H+&Ch2(UNO$Dx*849{MVSE6G#Uk z!k09Q#%Zu`l*i9#3`8_L`9Xx0+H)G1r)UT6hxWO|c>U>azE7Ds#+Szi-y;?&U{UA! zRU*pCfWwRzX-FtROb_-6un5WI6J@HJT|MWin^F@I1a>7C~p3 zX@Ynam*Yeh#(xHoIZ*}t4 zs=%RCxSx6$j~-r3Rj<*#ezGGySC`Y;M9RUEInsxT9FOuH$IF)84cE%(+8i11J9lnx zk^jCLTxDsAcUpn!5RH{m)o-$HM}tM3CJ=5gNr2|uUja7?+zaaoW9o63yUoDT2n=R^ z+`~f9XcObS0bFPrF1v5mZUAd(b7a8I82ccw;uOD7tdbb(<`0Hgn~PNUJReU@Uf z!7fEZz?0O`q#+bb!lsgN6&fA;oX!@84bj!DKCMN%K$9J&4-tBe?+yyh@WI|O8)tT7 zneG}KD+^5*uz2Oix(=t*qG#hNldHD>%E)cZ~5 zj|&MGvG+srdoK<$KZkL66Cft-Fwd;+pbw1ZVP?F+aX6Z$K}@jJglaE=EdSi*rx{J# zA>Cl&i6P9O!GxwPNc3hsKZU#lub3+UeYpVr7_WSU5cQ^&mg~m{lw>}C0?K>>6&>Wt z983rqhx+1+26Q z@U@q>J?ZTnM;Za<9?g;+V#bXpT$<_*!gD~}*j`krclL0AA#0;eR(VV9A4_hJY6)i# zSYCoU(&6bvvhQuzm~p=~f+ARvaEI>r|_OqFWV z1LAf>J9q5fvC?yW@2DPoR4R$L4$VKT#OJ)-qS0S)WrWly>j4F$?GLjE!gF*Z6}yTc zc#h)z;&+ z$BRnn7aze~+_Myq%DW}Q3m=ZXu&A}wQZKG~UI$X%r!Kv zOYdv)S|(Q`&9N4J=cev#*fEVyXuDSvwxZLzhgnjNEW!bmc`2~bVTDwSrGo@ehqZ$N z6bw@f>Nl>IfyY??wJmPTI@BsoGV4et;m`%F;=iPVT6Vf)NPEjEzOnror3v`)`!mlK zD$fs+Wzf5+fhGpn-5k`WaNm&F5KfKj`-7>H^p@W?x5UVuSKWWNB-dF; zHD8_IT2nE;Ov^>z`DC4K_yHkB9$5M1z_nPnoCy`6dii4mEdd%3>UpCPExHFXex{C7W2liSbz1EjYpHFZh;v|oKtULuOE?Q1Dg-8~m zQRwU}BHRt$CHq}}q97tO>eIxtm#z;!An$j})X!zVZ7yV#*~W@F+s3K~B`=RtwA>v{ z=L1dhkINQsopnby7Z)dw5Ba;_K~4@oI16s&d*Rkzq358ixKw^j7%5=C^Vq9f%WD+w z7Wj7z8hEa9N#(%-zA0(;0k;>*<=hM65|`tg#uxw~!|gX7PsGp1>h;gc)-vQ-BHxw&FC!_zZ?m2aUy7r4ewL@y*Y*fHJR#o`&+i^n_B}pvfGk3*zVFlpdWdpPKCqB<0K5tY=`sD>* zxP4Rjk64!q5~ zo!XbZw^#mszH<2Q3P7T|`{b2xDC%czD%h3^@uEWIXDj@_^aNA6?or_nkFG_ih%qYi z9ToM3%5z9X^Mvq9gzy~?;WrAw*oFvrg$P~=5xNQXd!cM#kJNYHd=rD}P z6Hbx{H$EP2ViZoc4L9`)H@gyUely%6J>2qMxYfh(Q=Q?~W8tUYh1+}yw>=EEMJLL{W(8B0(^N$i5z?u(K>#)m3=u_!ZpM- z1?HM^@m^#LEu!{cO9^{#L%pT( z^;6{YlUJ*XKP2DVrJA2#=)V&E6Ml8zYQDrli22nbub<`6Uv2sCp6YIgRYxt@{Ja-+ zb+h5%rq|BKE+XpFrv}2*g_HEvNc6s(LS#J|SV8?ZGM~2>f z{p`Qd2i13~1%6S3ueoq-HM{ve|A}lJII1IVj~daHS-{*|QKq**DvgT2ueufz;yT{Z zTgNvMH!hE+|4Q8abJ%(-2JQybAzXiC8(ekU|L;cfSaoDA1*kJ;Q#X4}MS(+o2^3op z;ob7*^50J=b{v-dyNBmuAv;caN9KXW-KP(~_#Ykm#}1w){DgVUZ18XkDsW`fefa3Q z_H!(m5B;?09>3sH+;2|SO%NyQC2<+w6^Da~lk*!*x^LnlGbDSMLq;rKozOK462&A@ z%sfj)uQ}c&b9G^p|v2t>^ zTPb!!0!sC#w|rPm!{7)Z>rZc*NeP_St%}s6y+4WhN6OScQ{%8I)DX*Cid%nKjBUJ= zrx?a5Jw9N5A|Vy|>cPTGvHbWntlq~?$>^5Dr(1>V^WO*tpIO=2p)Q^w0uZ6#CAx{F zSn90v4deE`d83odXwKlLM`%XsW3k>ik?2oyy!`4DQ*;n6XBkQoBT(J^E>)gCm%z20 zF=NDCHQ0xfZFAxmvM0b^#n-xHC(uzj__r>y=a!-5f+G|KDLz88IAMLdYF@y9+un@h z$>yQ9qhILa`N}ZD=l{G?&eyfe-ML%{=H>SMQ9(@4Yfp5r(qscU1>xbZ&yh|$0K_1& zy`^Po2<^2w2eCFargPYJSg~0?WMB3!ZduQrwrfv|7daWwyNuI2owpp25|ERbYaCQb zy!4AS)pgbd!c4@s!J9vd8DPN^9^*Lhy(%{?Xpkrs%SwF6$t73si_~1oZ?r9B+GVGr zgezV0OM5}$)}F6PrjLST1o)$!WbF|CY!bt5R55uu90@+Xp^+yzEzcia&08 zkVHgVaZ7IV#c(ByGq1Ed5#_d&r6$m&-a@c%jp%ITdS;sf^%}2}X#Kw7ZC7{RxB^!k%p)U=l zk((=Xv#-`g)W3p_!J=!+m=!?TIgY1+S-ggArEv->K-?aUQxIVlNJWE|UbVE4A^@Rr zk|kA!&?^4Gha9orT_?byC*J!*n7apww~Rl?fP~GI7A>vvssD-e{PFWWaR#$T{?UlM<7jI|txosn|J8WJ_@L3Ss^u-ZymrI+S#%G6OJ^&QxzH{mcs!4s1kA=a6DWGsU^VP5g$jOcOUeOfbJ!Rk!98`r5bg zdU!=c6tYqAdy*Jwj+$sd;C7iPWpnv;aB4R$AzbJ3K>2{g6Why(j8}u)1$J@DA2QrI z)kKn90Gz5BEKul2B4($=M3>M{!|6(J=hxL^RhTg9To)!G=8#iV)J(23RQ7AqZ|~+1kxEg6 z_tNXY)_xEs8!Gm8SwfyTYUE>AVsI(RzrioaH^Cibh^B{q?cfl~uM~cMfx!C6Kinv8 z=Clxyz;!E)yDiX;q*LR3|Fz^id4b@#z^Bcj+ZKO!j}4Ye-GiTXNQ(k5C#-8$aX7Zc z;{%k=T0BdGd=Bjq9~?KW&%wik=cb=*yaB2-=2clo-)YsII zaF!xRPVVsa9Q`RHreYI~gL03nZ$) zEWG4n;C4B-psiP+M-t_aA6Yl%&_;E5!_K40h;3!qLTrj({Rs|~xJC_M(Far#lLJ>j z1bVarM7_I!01DWFNK{}tw%OaHik48?L5D%zNj)|d)9S>2C0X6(=r?_`Nt#5xt&aGR z?gQS&(o#hH7Tl4|hSVbHD47#zId|$?&qX@YujT14{T%<4$Z+@u5f^`>_(& z+9DdFesL3OQbtqUCOrag19%2$Vul%|K7~&zzI^*$$lx3+McvfxwO!Fl3@JsN+%WEY z!Oot#OqLD=^cY0~63_pQheorzfp09xrMYx2T|D=311eN!zph#@2)1jrlE6zAyWk{Jli(`M-nS&rK5YG$hH;%#TLF8QaoK@S0)m=&75`#LpR} zTdqVdjz&By?;Z`RF#@Y!^EmVKS2elKS_mnqTcZGrOH>iXBo4kP4K*M9t^uSRpQa@$k5f2bX6AvmLm?3y*VzW_ z_zRy-FvS*@5&x1&iABc;9+@Zg_`L;EEWi}Oh5WXZZb^|Ubi5ebbzg-Nzp1gbr#qPD zfypRuL%T7;iRTYeycKA9J9dSi=%GVA32yx)!_+Uu#3H|$hfZMB5pkux%h-aI=h+q6 zM;izyA{yy}Fd&lP+;#9`9VpCOfft97dSTN7OW^TM2epDZF2J4hdP^mjtvIQ4JOx;T z&5gAN)(0LA;EJ%QBGMveI64U*#RaPH>lJ7O3M~;M7^1Y(11Q5{p8n)N4CoPQORy1u z4Cfg(9Yn7QBkO7IlBI)1EWyy%7{k*RO$paX^Ox$g%EDO=GN!iP^iCNlZzGlmIqpV% zC9qBWKE$x~B!Q+&(BTCF4~_usOwl2(9bVDVVlPBXDS6I`|L$}kp5=ZC$J@eNZ_$U` z8LFWR2vh06V(5zlO6vr`eK21nBLPGpVtTs-mH>%OKwtLo+k7{!@q2hMB{wG#grMWM z*(D4N5_ZJ25a-TeN3?fkwn<44fP%q7yf8APagHx^Tnt<2odd|5AP!Ar0GWr3zK})g zxR>-@0Dtt`F3APb!Kzq+dylCoNwC5M$7jz^n)OqS`|FW7)-ZLbhaovjWCA-#I9@7=~Gh}tEnOtf8h1? z?D;<8>gE~8me;1=xVl-cln*J+gSjv(CK{B=Y$9?H!91}8-UPT1n`Ey7$2mbl?1qf| zWWC+3Gc$6XV03&D@IoLz#9|c{J4-?XN^4x)(X{Z~cj9gpD?gS5beD|?kT|?Kf|>TNfwY4B?>_B3MsFGpS&zT&FS-pv4JS*w zZTNwMWtp~wS?ijkA@`Nc*a;(sv(yde?^95IST|S8RH+t2)V_QjD258Bh$y*~4!(3a-FDuPfcg>NR(W6b3rLz!OQ9masEi-dm{c>^B?3<&{m_^_7(ymMx8E5Uw zSYS7h7T3Ljt3@br$a?iu9>}hXoCoXoBR+U}xO(wz*9E#|u$!n~7Poj2jRmr4+N|-` zR4{%S=n)7>emX*GJ9_iqdzq!Du9!y7kI{uKY#IJ_XvYR-VMRU5$h6CzKr>Inrxc{l zMW zboGKwQaEQdKAY5+EJJd*zjKS!pmbk29i3|}k-HcbLAJumZmPW?S_7+5_1Mq&{5GN$Ru(-{BNDv{ z5N0{m1Hy}d#a+Ys6H*qj$Orht4QCf+zc$i4>&hDDi%#0m#%6I`R|d+C&U=Jjik%)9 z1pfyu`CG)}MCHkH9{zwF?|w;+Ki$83u;|N>zN^B;gQVn^tm!+X_{iBQzXA(8LGbHL zyJItAZs52W4~wD~jZ3H$1c`}CNq^9H!(GIW;+|QBdOQJ`_|ubotT#y%{Ey`Qj)f*$ z3#i)D#@#H>h=HdW-~eOZ4D6}fYrRR7krMEQMS%n&`23q3i%+pWVZS874z%p<;u+k> z)?S{W?MY;^#kqV@GhKZ5QNZFQnSv18~p_0@CrgN4__J2F}s@F1e`sJNu1-Bn;cn5&Q5Hc=|3DGQgR-NU|PWbFu4K zkTWh4jn9FlG&!GQK8!2jv@R$J=(WE8oPZFr)rg>>!H!%qRBygi=}Or6W>I8bL8|r` zagcZWz3VCMy>SLv8qF;7jzOOi}RS-!I$-Sm+Xjk|_BY+_t|r|Wgvu@UAG zkz%|7H`pR>l>)U82D!W0pMXC$4xiiLz^zg4kVxsFS z^iaJ-yv6XPmkApoB}UE|-kmPg4{v;=sqdUn%5s+^HHSpwt`e|1Urdl71TMnJi53ux zz#(5^Q#lzJKJLP(Ct7gLXwDRrkGq!=i%;XsKIbLI_OkmKAKp-M=?kXK2i`hw4p*%) zRr871WR~g9D&vw6t``2o@lV+R42XW{CmIJNh zA4+`j7^0S!_q3WbFDIk~_yPPLPV|Y2l|VMl>1$jH%i()C9CSRV$Ay zsGrJ2+08xi!A5PV?{3Hx7mscn8#@?ufHJ0rTGnH!=Zw+(n8bWd4q>b42v5WJ^!--^ zf8t|f)K7IAQEVJV>xzd*A6dC%LOto^@gi?Gs?fU}Z`g7(z}T{@z`N+M-ctP*kwCm@)L#&SjVEQmJg9%{Al(`BksS^#%8z z4+*pM-~RPqTh`O@>wL%0K#02*5Z8mvLYn%=Ke9w)vY^Wy@n>{QF3r1CJ{ZK-^P32{k^FgjZBf#*z*###~A0{#JJU;=}uE`EINoZ(xNE~wF5LC_H279iu_P!8aL!?cf>}K6&)@(ucd9J zxz|0N3O8|Zv?j7q68 zNi`1^ODdN9;k_oh9sT~0G!I zozel?j}HNUu;^{f4*e-Q@dx|D+%C-=eh7Z2Hn2e9W*Q)KuMT=I0kJQwdfn>qFf(T) zonM4ysr~JBEQ9W=D~bp6D*_lzC+q8R5AH7_V|4|`Nn~z@X5Ok-e?n?l$}p46*OLiR z{$6JA{bM(rW9~Sw%FU)@J48{afZ2T`XZgpLwW4}FQg@CBiAPb+zWQ8W<-6|C{4rQ{ zxF{vH50m`Bu13$CWn|w4Kr)8tSVJJ06AssJ1DepgJ8yC+E}=2;gW|#%B_}r)q4dZ- zh3XeIH}x3l`0M`bFAx{T;zHk54|C$~BM{IH4&P;<2( zA*+!?tU+^2h;l2d_qs_Q$_VOcwX;cX34UBU=b>YTH$84^+ckp5a-fH9OI=Nv zE1b8mxG7oVrYh4^mx~P;8L; z%@0_33|}DQY6(4XII@bZSV}83?o`rJDq2Z^n25%**2u-^i=0rSvr0}x@_0tOo630? zJScDj8n#&TY4HqDfCy4t>!r)@qh4Tw-dM+0{oekBR3LZpo5-6$rNMeGDF!kNqct{~ zQY-qX@)W*g77Njc_A9%Y7vGb1(DKGY=VaX0NZCyVYp(w|-6=!{kCxPBHHMSEG-^T^ zPXoNfvbXOY@>S`U%Xityd};dT~@#$d-{DRptS4ZgtyjX&gX3hk) z9+Gxtk>!EzcHELF`V?=TYA4KN#OmETF3DTntI}5(e4VSvt&dql=Fkbf?kzqd#sy<)20G0qTBFQ;(o$fdNze^RHe z%0T&~j>}0o&3|0h4UqcOi`r*_Zg~ueS`Vfjg~s6hku+)ee`SzXMh8B2^KuS$at5jM z&wj26?~W#4%)QZFR>i+VyIdD#ZLXW_`9%p)J92q6`*|`FW-aE{g>0#$y)Ttf6s!fb6Uj$>a}~>JZ)=2=9LCV8 z27KF+g>qrj+DmNlpUc@?jcg{r1uxTubVo4kk?BWdt(7-s6+7f+reSQU*3q=$&_*Ob zQ!useo#ev3;w(!eR)?zK;Yc?k$JQlV&2RcEDD{Xql$#&t`6c5+E*k<<9p4ZZk$H3w zB0!|ZiN{s&-6)@o`wBn4QZV>#i=zgd<3C@iGCB{6(Z|$)(a+7!_sn9X3(E(YXQV>7 zF4VZR3(Q@CDaE}&tCyGLu$SQ{ap<-qiH^+ya8h_B{HRoMU6^-v4DYnd)QsN5A8F`~ zGhN>@j0<;uBufb#b*;DfZj0F+Mb*1GVl=rDu9`gnM(A;HTcMd2XS=S8u{eayGHTyh zlkBKOTcgLuN^ws7oaHJUygrR}$Bz7UgBltB=03oOk_~wS_JXqllwdz>>z16_gJFc0 zw>A-#&kQ*l`*njvb3RKb6MHN_WGM?@N>TvfxG$_7o`2-{dhnd;b^*N+FwzMK^fp+E z6KO*;B|X8a?2zddK9gd}dOVzbQ0y4kI3z|6f@{vvU$KAjh_cUL(_{fj&@3P`7;?$T z*#7-6AWFsiQpC}T~n=VUy!KR($a(uYV zbDe*ajb6yG{^<;{&%d8EN#zhOle5j~^zGtmpe9uR>(R15!nL)0Q~q!NGorqNu6VN! zAc*jWk*Tu04EAu=x~hGdK8IWq6{#wZkWY$Q26GP~_10!9uB)r}_m7FyG?vSduq+mK^f!hkSo*KW;;ks1obku7DQ5nYm zZXZAXG8ZNm$v_>k#t;k}AV$RSP?bNrwt||D+3%J*;$#IAsPS*|RaJRE9Y0HmSJowX zdruXBu9j&*?zjk{Mi|1-#!>vmdEp6J}7H;eZR=cMlfMr;Sqv^u0}l zbA4o)Fd&aH4SD5d`1Gnht4{UV6j)jyv6*)n4ENR$|4cdSgA+ns3zQQFcO?>YOCFXB z_KM#MoXALpM=xiGlE9L%x|KtA1_&278^ou?Aevt+E8w^yKn0{9P=N$ZsG9=A|X0*u>9H*hayiNpWt@VG|~nk+(<(o$R@!a zB+u|(7k!|C4#CG5mU-o}G;cCv?uJ|~zy4>vMIfX4LR=fi&S<<__A>CKu$L&c%FwoE zZ1Tc!`%uxi_E`f-xalES(!XNZjDxm}p9)k2A}>9AE8I@nb|i|Oy&)a^CSfQ%nAJrj zyDJFvl_-B-+kGBLeC%;XN7r2dNJ^~(e7`2(56cF9TPF`;r`R_Z%V^FmM>`{OKe8V zseP!D(syO3Pw28sz3j^gtPJ#AZQ zAF6_m@Awgve9jy`Rmu9Jq(*t^1ujMS`D)IsSSWp|8#Y$I+ieDw>5aYQ7;FBG&7-ix zslEjGqnlyK?zP@Yezu*;mA#Vk2LUg)gIC*0zfy4OR>-dvS){J9d21x+{G_e8#}W zZkJRO>6r`MKUd%pKH!A`z2}w|ZGC*}frOV3olq%@r5cX$7jp1;RZP#3_>&huB^kg_ zT2Q%kH|}1=dx{fitPWwnPdEy4jfBSgB&Q14R+u6B^k{wZSg!MVJr)gqmQHbaL?6$3 z<$29A%pnkr@nv1KQA^T|0|tOiznRQ2M)K@1Lg)P{Rml@4EpB2D((l-gx$X$ufTgA>Uhc|B6mJr*;Ynkxz-@oThH z^4xPfLc5$I9|PY=tHglLsC<`-mbTSR)v*ZWJu!KNc;I;`FX#H*vSI@t->mUFL%fth z5bsHSTF(8`%sAB8ZYM*0WWu7s1bmA;938LoYLhmlb}QPEkEdCswPIXp!$=8SeE&Z__=a)&NoS>uU&f^#><PTpBVii8Rs`o7|F zex&mQZITW)y5pzECKEelr0txvOng+q8s55(bkZ3o;(IAMyeRoci_cJIfe0us}njuirYdNXG7@c#r2Aj<%F30g5_DgB=>+?^= z%!OO8I;gEW*}T`ZdfH+c>15eENt&Q?o0JJnG%Eadl$2s}A?_pcDGCD^OMNot!Nb1| z4D$j;JZywJGt;4mrG0vf{R82>g2vqg+gx=d_N=8~9a3LSBE4HIjU$MhPPX@_tlXR( zj!ipUZxTxnael;7YaoO$XhE1`>KAv_uq6mWK)kV9M&!zM*rZeYhoD2)#zh7Qe#^hf%MOf27hy|t}y5IsoM^eO zmksP)8*PRWmNQ0`G#%EP&(`j0r-jLfOUZ9LS^d#)Ev_)X+&zM|cdpE_QJ(QR9tjuF zw)vgolOAF@NVW_H7q}%^&P`k1T9xgAIY@_5wDPR>c}+?_o*nvdO4T`4DQ?_CfbxQM zqL>;mF>T4tHQCDHzT7$%$8Ul$fz8nnO7d>Z$2!~V;1)jZ)JK2cP7$8f`1AMrtK}Z8 zLR&VYO~$6jZn!wy{GR%1ccHH0;*f;!yR$XtrzkTE5e`A=LK-Zr!pKjm$J$v&|h{oYY4CMO)QnOBT%eX9}^9mqQP}GRo&N)C8k{BF;RUb>P36+Nhb~ z>|%`Zm7c|(bB#=qdL13_Cd+-*EtA^8t_3DttsqS63k*R^LqCQ3?n{}eSw~*$^aBEvdqVGA^89-PIv5iev|=lT){A_J<&>#RF&rR7(dCZKFS zdC!hL@OVKnsiXQo!kItQejZBEC!{#dd>Yhs5sltT%n2s zo?CKVmqaY&68+^E>fx~JzOPyus@@C+;~@Mkn=yyiT&Ev4yMfqa#4egtR3 z^5?`0VT8O@1l5{)UU2M3VeJ6+zn^Xu5p6zaoT{%nap$y#YPQPd({7nD5rf3?hkk7p ze5a4*s1IV=e`2l*?VjD7$uYaiT|necE{uB}>64!l?J};Z;Z=86Se zN}T?wo@p%Cw4)iDuX=r_i+b-jmJSiQ2bbVfYq$K5a5n=PVd3mxb3T$%`(iNto%^|A zc7~Ink7CSE8+ILU$6bkqj_bUYSK4X2IcJ`ToFjRtJ^!6u$;sVN5>Q#qQAX<(k{%7~ z;g;ub5Lmk-ti}EDTbMK0^txJowX!#lC0`Ox24zxx5_Y)-;*@pC?1zG`40!bBw}q9! zV=u}A?s0cfcwEB2ZS6Pu+dok{*3Z$?r*!yML%P3Or*M7LT@&+tSG+S*&;4g1FCt4> zPEz&N%*7^?hO0h7Vhtl2>aNryJ@fLY{|8QgZQqked6iNRnjXeKy+F+5>p?OAVIShy zajVe)kC+kU{iNoR(K{H7wb1VYK?=RHwyz}|)PN+b2aUN%_e2mM6`3E>$``A#gB66uriL9Hg9rWZVnaV||E&F(84l zQ0+VDWPzT6XyFd87*{Ui#cRdD1WKLNN5mi?&}q&h6@XxYxXVf^s?2wtt~}G1hbKS3 zFpNJ1f9}-o_j3`iIc)qtiq8F!>Fp0CH_z8|Me#f2w$oVdBGg; zgt1nA!#FtBhpSGMKha}8ZJe15*%KAvS5pr;j(K)L8Y|LQ6*_sYa8vMfR7m z62&qhNG?cnpU;G>$l;|d@OkS55;-(NXXqkIO5I};L)~K-!1Sq}60reh5^s-$x!t!5 zZ+F^3Nn-pQf#MU_jV~qC`j&ZOCSGUSQ+AhVFd?aGhPObq28V|!UpET&7SXS>_lVf^ zsPObhq6aYkAzQvM*sD~ilhLSCWQnzOl6{%QnX9t-3-=mlvCv61i8Qkgj9r*MIe~dZ zd%@`KQDs4lo%i8{I{P$XC$)p#ac<~Ushr{gd+0(k9V`*GCtAst7fu+kkC@PdNXVRi zkM~(?FO(2Mzc!h|z~r8LP|h|TnYbcX%^AQ-WBJBV^gi<)@z$ch-VK)`Mo-xb#?_Fg zrQ$-k{aB70nS^w9NBfhkMm`*U^!v-xl($~(kpqKhx>Vh7Hw z6DmT9{rx7%cAa`bI%n2ah;Nq&Lq`jm&ACjwSX2}3JfNo0k%D~MWT0R~b-P`F_L5pU z>Pro$+|RocJs@H>LDW9!Uk%;H(9%<|<=#A$-P4HOcb>?fT|=M=fn*Pm@Db$8LZOl8 z0XX6O$<*~B?F~Gp0~Q9%tjX*qRvp<3n!?0Y6J}5i`bhJP8kttSN2_QwyX-&!bxU9P z!q@Mj==i*3`)5UWuuzGVWdA|b$2vY`!~9@17=7!;QbPy}mt zlAtmBDmN&eJ)(7{USrl|3ew&aq9hWbhwc}gdPyM(TC^>0pBc3m`X)&85o`cmQzDbg zo1I=t9&gB?yce$CZy9;4a+0oq_IWyLDRS{*0yNnyItin=+Wy@?~hBo>}1!U!izBzNNo}tpBqV_+_4(oxQ9{R6h#J0c1Nl-+wM84|yw}H#Jjuk76T|g#MI}l*z|tYl+*PQc z9{1W|EuS0eq&o^iY!s&QUYl~cR4!C5anB8vEu^By9<9e&=*;H&G%XKzEY@B#Hej5K zpB+DtbHBS}EDN4AH+}0Th1FNO=V4*{WGztl4dtu2%Zv_-j9x56;67<>MI%mjav<2j z$l&*3AfXmDr`jU`0WybVAI`}n;1d+zhS|af`(Wo^u+3igh9``sD}ND=tH-onv)`ww zXOK)hMy6dr@guXu>7NZ1m&D$ZQ}$fA%Bu$=8{K|IT@|mIy$SX#bOd8iP~BPKLiDdJ zSl41E^ZOHr*jFUmd;f_FcNGBTh|LRMd>$Q=M^W@W+#@L`ps2}4XX#x5@nH@K$;=fF z(i=Z=`kQEI!fMS3$bYD_wrIX_Nb&A8oZTrdWiWXSEC9MlfqPbt#heKQQR(i#j^}Doct8Jp%2lI9*X_tufY1fC zn)7QSBCZU_fF>h_!wHjyxnb3RuGsd1Ke$HE!p5zZ>`3u6N<~Po?P=M>OU&DS~453t#lpu z(n-t@{psh})tzaL&LFWz%D)EiuJk8RKI37ouqI4;VV)QPsy6_=hr|t6DrmK#Q2hN~ zE(K3=?b_%?BJd+Ri(Pmx=?|p_vQIRq^y&oc)6(=Lfm;WRb-`;6J*-l~Wxe7W5L{0| zHf~(6D25{lD*=KQBH;?r^a%LhD!DKfut{wGnbMw+Jm<8znOpa-e2kJC(mp|zGOO^= zDPcUy6QqgN2*2IUv>0`+un~WvalX^e$l%zt@UL($f%h|)q{qy?b3TjhiWx|m_DI}T z@#LgTT+>E$G`dw@Km2_b5Bd76IB3Jpa1q_1m0(!w=n|C(nqNB&ZHac1EJ!d_AiX+4i49TR^;LW?Bl_un(X_TMxmu z8HLuKz!^;}8ohn}GP+Y7dRulgVQcyQ8W&&W|J&OAzm}CpxTg}BE#II%zQjtOjPbIT z6iFx4M+t*>EOm&Fom40-(hz8O^HdE!r))?{1ns_yjng}dY!Wx#jurtjmsup>8VWqA zfdw51Bgl0gR;Bo2LNl;umr%-ElFNtj8F!^!>7+XFakF*{3y&ykPc0Z1P~a2lx{SAC z9D3)ifoG<2Pr`^w;^Xh^?LVMRGr}?SSj;3~Z~MfwyQwu=6@ql}Nn}VZzz2VJ!jDiH zb}Tt6>tu!J13R$mBF=vW2l-}}IVqEQPe}~N__dvOZrgb4qHGH=v8O>N(oySoI7N;F zdo#r@VxtO4%IldZ*9`frmjEpQ{wEOj3ys%OKg;a2wB+$&gffYl@6u5dK^Yn3;v)1# zMVZ%IV)P+tWrpod-=H@R5GoyuozM$X%R)h=?vw_&UKH;)RKRwi&-Hn;)EFvsOp>9L znW8EN53<~n`>B5p%kepz0DE>96!l%!n`jD7khql*>CZqJf)(j%`rx7q=4Bd29M}>@ z!WO6$b5;CjhRMXX;>{w%hfC%@n`R6}hFmQ)Zqbj}L{MupJ;H_3Py*mE-HZ24B7Zt1 zdS_Tk_z7^;C4rIK{ef>tV)3<>)x1P%V{wJPI)p7&)vS$+P#jk~{NkP2_=BYS=9n^4 zJjOFGQ&PwE`Tn#754}LNo`GzQsgC%T_I@ruIn6U>o1pTk0{AGA?7#zcq+&lQF%J0a z(NCfdvklc4xoOGnVkD)8qk(x3@~Erj_Csk_s>JHUf%InCx5=qVNxGj12IBfC_jU|U z0H#%lBFqrJl3-U(9(tn|jHH3sXlw)rR@!c9F7`~kT*pVnc1TuxAZm-OAV4}~ zvZQs`PC1c@)o)6Tu~zuBEV)hhgc#Nv4|#oW zUrw_5%Sc~(du`H=XPH4^?nS%>Rafx`S_dqP(U-P<<@B9xHDPFGRvW8E48Y(snv3G! zUvu|+u597}*2kk2l*AL0be^(&yy`uvlWAFD)^3E$*OpFpW*dGXH2Rk4ubV(=IG&5_dHterJn3U+ZJADcw>YDaG@upQSCA+9w56D z`qwnh)FDzw>y%Bpn@z+IeB&z2;p86`(OR&$n=m67r8#=QMGmNPd8PE7w^YRBsq+=U zr@svZ+5UB;fa0yn0KD}w3*7V$wbOs9pg||c*DOGyiN1PW)JKfvC_DH4^k9>O2pVH_ zC-GHRV!TK2@gP@Lv{Z|LmT-USO?~M3$bj`Qufh_=JRU4C$uV^LsI^^7Vj@|@(M!DD zZHZ*F|Abmhw}-$BOaSxb81wp59>{hCB$-pWplppvD6zyORVSN-9}OHWX%$ex_^il( z@pRy=Dw#*~)f@&*>j);MBrqEhrTs?|<%wQJhy@T+u6p^O0Q);%Paosj_xYoJI`hKn zpr_l@-ZZ0y$`o}Onbv556ldd8%PGQb7QLRB!OYC^PLeC$$&5p=>Z=vi>?%p~l;}~@ z-s?z++u&}P4o+RPd#Ca*{jkh5O~_M2dlh|h$+0KdL)2K|FvG_E#HX9ui-5tGUm+nemU6+y%^N%nv zO5>tuKB~07qgClY_-~Modm-0~_LpqVImrwJ2KbVV;s?X`IUkkD)(FN>Z$%!#ssT-x z!uUN{SmK9o#t-ZcVrVc54cn-n8O)3Jq;6?|CM_(+fO$UZBy0tn4Y&joOb6!5QjCn`(ZEQn@&llGI^jxp$$(#XQZ7 zmZOuw(T0?Y2UQL*&e7Hz0?q1vIv-Rj0*BhN%0uEvIrZZHI+G_rTS0M*ZbyCewP{=c zQ09yIp2wFw7nRU;v(XhQ6^Pjvpa;>GHqZJ~=CP)7KEf};BTRnlRIFsb-qnzsVLFSQ zX5?+@qa<(FI{nL}ti1yyfL0e)Bt^*?f*D=Zo7A^h3>%8MtJarw@0Vqo=acv$zK&b3 z_SK_@-@Lf4BIAEY$#k?_jan1{J*@4?DM0|xcgR=n?1+9jtaUgn^4NAaYx2}LN6q`w z(L55NaE&heASJV*A|pXg-wU>@a#>ot%4U1Nz&!pqr0V3-*=EpzWkd1A#)oMc6_|=! zd-Y z-k&)cUe2dqrXN*PSPd|rlq@oaC<*oXhRHP%q$MB~nlP;vf)DYoN8w~r11WZQ3&dku zJncn)sJDSmhJ+oF|8eH{ku|jCqVIzo=?EH8ydi(m+yN1$lXoZedN!(-RW#U6^qPBV zfGd-u!?wHqFWB8_XtCXd?3);j%CXY)tf7fAdTq}`k-@+s+~v`;7{KG!;7AD z{cDE}4AL|{+J~QUOgzysw{&y76;}yvYCmsPmSU{h88G-#zsHmgJ96#BscC>Jm@CFo zSyXD;*QbCbE17S4#KXkZ(2qA%jbpO>!>7hIBZjd|E4B1Pg^~RGvKs{f6SpJD?Sn;H zH$0_c35(ZrrBT(S!AQw@A4LjL%FX!^B24*FzM)aLVZlJC5Wx_!Xlb^X;Z?8m9cCui z0KK=-@k5(|ejxE_B@@AUGpi-V`qV7R0@Ef`=8s zt0k`#8qi+hYWTBJW!^ATnof9${HSV-9JjFZ*QvFX=L$JiwuLAp4dc3m&MTcM&OoO= z$EAeJd*k)NkP@3gsGz;`{_oGI{owPH?(ZTeSng}T5~bBVRN{bE|LQ%B@DK~Ulzsop z>oQu#_R7!h?;Es5DbX-`VdxN$!NYBt$$HIKpB(vD`TDLZQanWv{Pcl&18M(AX)9CH z4E5>wnEKmBP=Y-$yH3S)LXz6eQ)b{DOX7pJj(fU*Qd=zGY>AsPbemuX z?e&o&goR0URi#d8YO*Gkp23JO@QOxxqI{&4pDm5qj7lia$cN*_{)*M=@tx>7ei-Sp0u45vft# z@=|Iyu=oZ|L+;D+!zy&x9xdOX#T#~Bm)|I6DD_5&oVB8rQl*K0u+R`|+O~JB8m4+0 zd%(PM+g7wW?7#X#fQlyB+XU;hu`p z;(V15m}p}_*~d3>2ED}Q9_{ zC&X}T1wmfV9SJTlCRFQ$#^o>KeI&?DCDk^z#n>oXg|BECW)`B?`il#KE*Hc+t`D$vyPt@F3d?WM-Xn4mM z0jOLjEsVi(#OV4PprUB~V`Xvy6|^vKMLU+5#jtC3QP@b?d4aR?yOKU@9`*M5^7d6e z>t4-nHP(OCyr^0E-X$s2#Yy3d2AD4R`rgPtQNrK4xGT{%iqt4KlK9Rxoe6>=1E~AG zBb_sGr<78DO0xPMw+Gy!wtDQ~!3cfzlzcJ27SLYu>RQtTCs$u@m-^rb?a-* z@z;^s0fo1bia(UHP4bPriM3+yr}GKSS!r#<57OPs`?aO_xKZlT9*Yp&!5$`C?pAGG z1Ams<&~ZybxrHTOVfHNF;9or>&*BSRf8V>AKi7vJ$y$0SYG#7-Y&QFH2&Qw_Y6(UI z-QNTq3^IwTj7 z{K9Kz1MKXDz&UOp38PhRUhT+au=O#+`8Xn@lin}buVixQ%`>~=;{MzY0a|3Kk>yV zRrM}<1EBWw3p+Pnn%|ku&`cPRJldOZwJL7=O0}e@uyKyokk@$1ZU7#-S$iGyvH@T>AqX|X{DL2B&IaJCHVvM-y6e7}?|;Q^0)9?(8fyOS za=zUTuB4Ydg4}a_Tj1F_t*{suqgs19ux`RZqh_oix%E=nNrO@w&JHh0yyMXQ=Dx1c zJi3xh%i=C1vHO`uW}KCN$jR+{7GL2|!>JxoXs{{PH_ZNIsV4jd=IY|(+3X}vYWhw?S1X8*G9Y-s1lIQO8sxr7g_(l7ZKm4rosV<{9|-$oOU>` z%?T9>9NNDUCE_9_y|{H|s)SI=b@|{mBCf?T{~h>_F1q0;Su)k~;gn7$y=xhsyCN}T zjHhOvk@Fa3+peT%o;?O}&vZXor0<7Hm6*@_=OIbbb6zcs zs3vHgeS$ZCVBO%Z@y#=BiN(JJ#6?O7J}E=tQL-{31I5-R*+-yV=9StcsC8*Yd~zQ| zmQ6hTSTVjiqX2x=Y+%281E9I0SX|jVW$5WLeDjij^dd&<{=L@diL(_JBYfFhfk$6) zYN^hySmgApN3VsU<*c|m>0eT%ni9{l$WMsjg^4{qJUr`b9O6@YAN>nMRYUsWhjD|K z51RuDQoMl*G1Xqcg2?G6(bZbmk;+rvw4w&Jk!d$VlO2HC26G|eDC*Mql zd3p%CvP6up@wlNfw|(@RMJQ+6Wk5prOoZ(+Ht`V@-cvRS2eHiZf@|3x>J6?v6#L|reDInF?-*fX!Kra|`RXWk1PURR)|ZXbQh_5b+G&XP9mNwL{5ROFNc-qvsY7<8t@i zoZVj~+$=YqXa!a9iuQkIeF*JyxR)zZD3zh*cr3OcE=)1Z;$WiR%kGzFbDY(BI&bYa zPT2jW7TkE!{z}WOHt{BfZ>I(9W&JKZR5*Kh<;UFkub7!Ef$z7%$(%dj?9%T0#%uJ= zPmkuep4VL|Yju7mv^a&Gi3qp*3S)GaJHAj+?>1kw{%_?~-1Ck)lpN~X+@HDoD6-d0 zvBe#e*uHva{l_oTmUl#?13RPx`hVmv<$+lldUssi<*>1#`A;Sgmr;+}%RaBC_OjBt zNO{Qb?@cQKS)%W6t6Lj8idJsFJn38twX0x0_lpa7ITm13d-}^q_1GZ3^j*_NU6J^& zww;b+(m!sm<R0Y zGunR5&s%pPCoOz4{RT$n!+%NSfYJ&jE8mm0NfT*D^w@?ui#Ylz#K^;I`ljEC1y9_v z5r;K-f9aK&a*smXkumg&nN^Vcl;xHE{N%E;#nY3j&)>+%--QI8iNJi}&m_?Q8TJ_; zTZz1qmff3@S|$>+eRJ6RB*)Wm9pZ|MFd&hywnwN&Up}6AaemDq%b92Ral2_F?uBcZ z`AkDR=Ka0xZlP!2t?#*iY8Kj;GZUM_^OQ@`*C@C$k2Ub|vBpJbdFKdFPMggae37ny zf5iUn>9~iTmRHZ$G`c8gbo|wfhX~-fxub1K$-isVK^N{_BbmW2frk2;|C?yvy)}Dq z-ltq`oV$Lbx;_7D>4=@&g;hw0_?7WrT_x4k-p6yt-bTHBGVTS0P~YCl4DZSCPTfop zIJbFMYj2D??Rnzs5HJb%>Cq3*-jQctW`jCkmmZ2rD9H)8wz#=-y{h^Z|MA#^l}^i~ zgu%`-O|^fU{L4G+1Lb9xTt@#Z5&f&PA~*7?*v5Op`V>#<{2zRa#Wxp(uj1qVQw?tw zO|-&Vp&<`<>wAOd?tiJ5d;9x#ZCe{*x&s@FZ;O5Ose!WtCi&St{x9(!%7`n@)BR${ z#pXR~m?vT*_Vrw{9j_5Z@>NidA zH!HO>HL~{oL=x|{)hIR7HX<03FYCt@ScW{5v#fs8Z^$atC9G6_;(MFfRrfoj^BhZg zPK`Y04xS5-C*ESW(cO>2kY7j>t$MGu3-LU5$qAagJ5_xwN1nB16RFqSVU!oxI1tn^ z5X>70nHe}3@%ZlwS;oh>=?~9-!OmLN-Z$O$xy1m@**-97z`_z9ZfQ+j9f;`|jO7i+ z`RM=7wHKQnJdU(g+889v=$6S)0(>YxNj6H}R^Ax0PaK81V<%lgv1p{wctdA81}|?9 zBz_!HN*eqwa}9CYX7~>Iv+ZzFc%Rj@hRloOP zGsfZ5M;_bgs$N8c`8Nk)Z3;*kam>3frQ?CkZogV-V;_I+ac10LJx68s>6Ns8(zVvZpt_A%w4zs{pdgNjr5RqX41Wj>*iCxhDN7#lezbsf`)Z_4NLN> z#aL>WN2~{9MWDI+{suoD5mNc}X+(E<=(998;}h?RrUlaVNel8(q4L$SySioJ*gWTa z^^qoEn97RyzT-&p^BA4F39Jk2)?kanfBI_KIX3z`hGsdW(QZ*+5Qcl=Qx9qR7Yf>0`5bWWVe}{x9>6*xKqPm$yAN1t0B_C->*gyRnpS?8?~n?zjxJ zTP(`5`JVclX?v)k%cV%fe1u7cto!9Vt=fNDg&|{b{cy4i8-1-|bjrQ)M}|LYHkD=yzJnrG+Z-^6On=_+`B>_?zzoa)(@y z$DGwer1Jo0%n!xTC!Lz6o{WZ7H=HTTo)*+?niKJk9T=Wb@cON}jBekVB+TiziW8g9 zFL5npY{r%CU-2eolev;U`}6+vBNE|(Pe%WK0ubQE8lA(Nvi8TnnaGl#-EMHc0~uc5 znq=@qbva^dF7ox7-0vOciW#I2EJawZV#<2SjS0tuZOP9;H5tlQZ;@liWKk#ggN>-{ zF`#?Pf_hx;ok&5%FZKM^RL5TAogSXSxUAY&zMb%E+2M_yAuSq9TpeCuu-|q?HXgW- zK{x$&%KqqeM8ijbDjRtyTl%$&%Ykm7uDU@jt(X~5W>!oO5YF`kKh$XhAuA?CHt~G1 z9MM9i&MY=NN1b6d^ttmga*yu;P{732`R5a#teOlfU&c|HmZxs?2Q$<+k#3dQ%WI~J zwLpVO(5p%zyEy?JOLqeq9LXME=4yh z+GCy1-k_JSYv08*sxN12U1}R!yU?R9wXlGHBz&f3Z;RhtkMn9V0~(-EzI8xV z6mz`@Km2K2_u$jZwWa@k1nAH%I`HwV&x`O26U=hKs&Zi6)HjnEDqWmb`cA#EgY6h;Y>HY_Hy>V8`_|g-!f2sxhFlOnx^UlMzbZG32dQVrQ9xYqzE}faXzPF}m(>Zu>tyj5(2c z@Oo}orPK4%_OK*hmmVMy2Xd&dOFR91r4JD0J2||YB|`-(n%%2GRHv;Ya8Ur{rr#U0 znyX`#$F2*>(pXun`BeT+tH7C;o7lI5ftveO`aUSj!94c-$dHJs{X5f_t?1wp@y_)V zuf`Gg4>p9}K&4Y^fwvbNof6~}qtZ2l?>A_}L6j*!H{+Xy8sa^}y-ChxE26Sd5Z%RV z2s0pA53={bt#E9U>)fBel7W}zYxV;vDCxRc24yy#lA7+&_J8X2n0g3oOl@@%NtgxO zMWrje0UKox0!Dm$w4Sh5{KE}ic-IqdArYWRX5FHMC6z^wDc1P8G;WaYf@kvT-~Q}d z4*TtNVx+ENLq++r8NSDFxa;k&g|Tm+$Ez7EKFRFf=)7;4%fCi|)Awn?);aI(m4km) z==*^u>lEtv11yQK`|5=fP#nlI05sBC2TzEwU;`3W^rv z0<0#$VikgeB^0PS%Y=yk{u*detGpSstQE10Yy)X?0B{}nO5$_4&*W0RtC%kxM`gXS zGM?8wgv8TPXx0J3<)tw|2b>$qKrek? zHR3<*JF5hy3p;!a`1T#oxohV8RZ*b^jUIrp%JJT857rFxt99|`FkeW_uyP)-RspR|Y=u&}U zDuf^(^9MTDMaNW&Ty9x&*Wces@SdnBm%a5_FkT83SOia}T&;FGFoOb-%mReC=MiQkH9Z5^;NjdZUe8uwPrvwGo)BEU7g0l(NGcA9rZShi){% zIz+)I{Spv!yz;}V=`wr4MYRY3$cN#Mh|uG`daOWr;F{kNha2^I)UTS-NQWsbh(lLr zM~Xok4A>}BV!B-`ow0QItKf%ujexz^hxZ1gOF#zE!T1@~+mpZ`9|6-)u(nZ>%{pA=2>~B^>ixcZ~D^8gqF`eSE#q0#- z{t$stC1FQ^rtGCsRnUjU@|06junG{FxW>HpS>X-DlEWC&_3AAMBNYny+vjVvrMCfB zJ0e+^e!%TGV7gqC*(`)GEHOTcz@1C?Tni;Hf_?1|Xs%J^52b|YvxU!ZQ#e@8-W@ZC zsqjU{QV-JQ*rfPM%8XcMCO%%*?=u2@t6bqH`s;B%5V{O1C%!v8bfgv$Cej#SiR*`hK`9Migo z+RjITE!)87OxeCmY-BCioTe}<$FiLTiARBnv!Lh6jp}9)VgSTE3bgjiL%l6v&PW{+#^% z?Bgw>WO_gXM(2a<8U8zvG!>`=nCFK9?K;8pi%TNq)-s{XRhUa70Vm98{Qs`riLW}K z8$gqw?R4x4oUw0UqL>sah+aEU*I!Dx(2OGVt%gEj{6!;0rF0n2Kn{XvH?m)}LCEjF zgR4jEIHTTQ zFu%EakAyLaeH1L*6`e#;pA^wJptpO}>w}FY8z6*E|Dc$Gh&M5eVmG0MN-*Q_s;MOf zDNAj0AO6n|F!aU3RL87aoXTt!Sv6=-_)2>NXEsE-Qh32}SoLl7gMUi8WrAo+>FeTk z`h}o^hN|1HlZpEMYlcPvuKK)YFj$to=&`{1@V3V4_!S4Wr{_$nwTznK+s_15PYn)9 z`Rj9`!U%YWgOY0!u?TO$dBMfTX*;koB@Z?dW!cpv8F3i{4oH!S;)298S37(K)btVt zj_o<<#K3Bcq$v-7y~H6wqd~a2A43}1Gu_YC({P8Mu;5=s6~h;6Qm#U zs?x8ybrB{ZY*feqt!AIHgIRPCD1yQvPX;CSa`X^6zrtV7s!n63$l?~fLPjkvj}LT| zv8bRz@fq+S|AR;}X~-rbhzrGulrNdQhY+@M-o^r&F8(FwfFGq>P+w->{^Fgw09PnNm zll#lBt>2fA=|G7yiGxhMMC)LQOyOS^TtQoXb6M^ZHE&78WMt=aj68uU5cskF&!nK5 z&#uIv_W7hDKYrBxsx#g8C-FkRr0&hoQ2Q}*X%SP zzz^|l;H4ThV#&6=`>zGdY1|2iRQq_7IJwF-m;jn2!x6k91RO6n zWb9i!PqM5BNN)gb6N1rYsm*2R+B!6IV*HbzA=9Arvm!s9K5K*F;Guep$cwe4eCu$w z%VI6-L@bYvOF^ODM2R0wbrbM{HGT>Vx4=+=^6Xl>zipf>ggP+2mnQ|Kaz*s-JRa!4y9})&eFA}9^=|J$BAUL`LETGEaIT-`Maq2t~X%0{%kKrkGZ33iD z%rc?UU9XM+uy2QfA@B5Ihi5^8JEcI&HUd~)igd%Ast!upn@|SuaH>?MZ5#n;NXNnq z+Ti-|rvO|ns}Ua!^+0m~%5BW{DYJ9~G68%frab0;8AYoNm`|G<5iGzG2)mm~d}@DW zP^4?BlSA5;*n4i(`sh(x!>XxjzN+!f=&@5vwWfTwoG^|CB_pE{7xsRY{7%S9?}cJA z)%}w!PYC7-DvD5gCe2ybrT!8H4*s}V6dnIS_Yp4N5Q*iC%oQ?mE zewtelO!HE?mqI<~bSnEiUt8b&`g?D@Z({#T*z7HF%W%WUl93g*DORHQ*rf}#h4Y8L zmHZ|-9GtI2Cp1q8CVpu|UK*AUv=kHIie#xVJ*sv$BLkP|A{w6abFtJxx1<=o!tWZM7^Ppn?=8990zd`h z0l?XtSB;ASg?iL}k;YjNl7+9vzalU3v4zw)o(T8}AFSL=#SI12Pi%54uYM0L?T zl`qr-WlldTSo1DW$oKMdBV$ntmHCqa6L1|_(JqI>#9hrja4)(FfAUbiUmL!>o9%7< zl_{SltHdth95`Rx8WiVF%Xa8pMduTFD#Nw+SwDnr7NU9Po`x1#vt8(u)E@_h^(_#{ zNbS4LL<-)A$p~0JHEELGHr6euQ!RSq=ugLk0h_TbOWWD^*XaQ=hA16v2Q4g7S5N1v z42h5LI|$~%rH9XEQ;e8Gl3ntyvP_HDq%1g-Z7PpXYVeul?6H1Qm2?v_&ZSVQ5B~~v zIMZsZ_if{aQ5!@^-@^RJ1D88Dfd`?()RFKu1KFMH| zmi0X@I>V6zp_NgsXfYO|FTk*!juFIx9FU4SL#`e$?0y6Q4(F^u{%xJMFXQlkbOtfy zGHd%VD}?gC@CoI1`pw|)I8QftqLf-&I=%#tii0P7xHJJ%PM>v;qq5}A8Nh8DNK~uw`{~Q>u2WJRn-Y&3-)C zLt+p}q1gK{mpHU^GoO@tk5&AfWBPr3l&H3Od$9`-9`e7WZ400+wS^5l7RJE~s{ z2PvMqkisdPC*mEFh>5SW{FGmmY%pF>UwEMeDqauyO@!?CqzHsCWgKCp3$h-_tY@Bd z*TzD#zpj%B?5b0X$I_qKZ@k6ggr!3Iz5obN_H&HbNVYR0lZ*zB(e@Bo&oNPTP8hPOQ~`biXlF!T)Dhj4noi+y6!8po zVTY!zClDI(+_7S8X;JH_U`->5fUHp8@|39{^1*ER%0}S2=Tnf zM2HbkE)Iy~a747ZsEwR^jc7qV(y!!K`hQ`6u7{qU9DO_-`j2rWAm?2`u?i=o)J*s? zx(y|N$HDTEV-gDgQn}j2Q&Q+NBo|pGFj-jE=nC;$B(~9I7Pv<_3>jQxXMlTm2t>6Q zDA_TNm32h?wUe57c9C5&Q9%xG$?d_*Jo5kqg{R*W_8*&(7Tw0W{q;Rs4Nwdaj=JtM zEN_^%cgoKg=T;11F{-C{(@+S^`H(Tc|pVj^2F5zQdN zaeb#DKKbeZ>?+~r*fZpB`#S@!rSKP%9suD|;3$=`);Wv`VM<&ZUw6#_u{{p1L+pXC zrIN^$W9bgK3x7EchhGa!Rfi1V#uALko7Dg%oRXteS=lZtwAXzBH>R>W^|7>Cr&uJa ztE{G}(DjU{L_psNJmlEem=Z5TX|2&tr@<=3D1IR=7f-XL6suJYtJ ziX0T{DzpQO#DHQjZJ-Q~$cA!+9c?l2r>-etaySLSHXLk^5VIpG2CxGl;{i%KK zt>Yqx*i^HiL67a8DOJ6sVkTWj2pRYZjr%;|!LD&wMYc7+tP-5|h!)*9>DJ_0MhM{9 zgaCi6K{5+VrheKFg!rV9)5mJqhj0T}WFa$g@L+Tm(UM8)gdUo8kHUlgR%GVny`~y1 zkI{npeZcIStyus)fP#^24AfI_?UHFZF#+ylC#6*PFh6&Zo2suNvWsP5qF<_q95o}S zGJ+0#2oRDKm*Zvfwu(cP55!ffyOgy`MuR>{S&DX^YkCx+ai^Y! zxVedtaizc+AZ3ANU3MIY>j4MG#Dxiz0atxEx%$B$L@tRg0Yu#;UGQ~~>~zszP2~pD zGwnj&Jog2IjKSzEUKjTijYLM1CIcEM6C4psdpnPdgccrOki$J9d%*5B7aM?04uO>D z2|7iwQD;P-8!^%D+@lLsPI)}+T`eaM!Taw#hT4L8Z<-w-k%<}{Od~Q^j5JZ~;jkg> z++9B{J8&_H_@Y-*yxKmf4KI{TvEATc0^}~+t9f64{(}D6W06R!;ZbMGBDOL@JAr5% z_ZZR5j!U9;a*wr(gQ+CdWkBXaWpKr?Kav-1sDgmf492B>9M|vt@6Ys=7k^qmZrb{$Z0vCZ~_E2uBkodwNyLVLN zi^SM(`{+`*EUxcQ+g`14)q(1Gte3Mpn{|-<@gP2V? z$ut55Z~%ceMnMAo-8xCthy=pXgULDA4UWjd-X!=Gq%;N$;ViBe|BS}nT=m>7cKUsv zgz@=~2*5#&IM_mx?^hybuc*RI*>C`OB=@6du5uVr1{W#_0EsomP!&Q60YFwh7uONN z{>)KqWoNID$e41?}YITtTq=>NJR2DsKOD&fMg^@v@MZB{VgImzw@YCPpIcd zY=27JNddilUaTH4h%U*OitAhVN$Sbt9D4|n4B^Tr#g|z2so*JZlSX%C7gRTX+^!t0 z(ocw~l)_6KR4azk8$C)3=Y3lSLN$biDmvUgGa^!WZaf)9+=~4@NN;;JUVQd$T;hV0Q*RGQ5+&C zGIAFf&Vsxg-n?od&=cdQ#+&as*q3U*(J`xZZi2($T_WfKLy?fosOYH818Pao$C|=2 zh4V^<+z;t;;5@()E2fI$lbXp5%nrHN^FXwML+`7rPAfncKuB*ssPExe5}{93FZkm8 zczCj7Qfd(2%%xlgqEjKrOMoZkZ$h#hB*2cMoT3DQrV=}S;QsR^hvE)s>W;~WaC!SL z#Pr&)YOWk_11L{5pfi^R7Z@tLK_JGAa&V#v)f@_ zu+9O5hbN(TL@XV>x2BvAIUOr~lCgSOyC_-h2ml$tz_oGjwtTg#uic2n^7{e2KA)bBK6Y$fo zjfjM=byq;1z8^~48hF|XPs2Z%Xn1)claiG5`cU5IV1Q4Ntw00~5*GNRs5;NC*6Pwp zm|-3E^yjGXZHf7>JcakYQyzmYXe(%PJaQvS+~n`%sn=6Pq)_!x>W{Ih6RslN$M`b~ z0&~v@SuJlP-}|N0zZ?>T2pL(;U*I${3Q2yX-2ExaCZ*Z+eX=UUF&v@!tFX!q0Bb(dR0cg!jdhNd}0RQ}|?L9`IF zCUC?E)GVR@F-#b<`?Rf*oC$*(dY~#8*<6@~BkpGw1HgdctldZHVakHy${zK+Hn1}6 zA|Iov9EJ)fLsjCv*4j#JpPjc(g9(phJQb{SDC)CBltg=9wK;2DW?mNKd(*!?@_Bbz ztpDv$ZYI{EJT9nY?~2Aw{Cb>nW9~)L`V*%1?~`Bq*-O6? zhf)v3eSVq$E!u}1dZPcx*xkkUAD@2So;V~_YTJx?+jRNBwWPfd?|=VxZ8n+8>mE@u zWqzf9K${3U&Dq+Hog#%;39V(#Uaued_wGK39ByCVth}-RUipV(CJ0od{(@qP>)oyyuJI0VZ(+n2l7-}`{wvHR@&w8W4GRtmPz#4v_&m&-_ zjRM`iR_rq!$}RYwSA26(2e&ax*;5;b2Om9)x${Lg?{?2}vzWcvhmy~Gv{!RfvE5=t z>5ebIf3<({>cz~NXiYf@LlD{mtUT}h#1JEScfnNgn=uEE(&sLeoN+kZ*!-;YZezlK z{Cg{esu-7+^6?jD5t$EG&6gY{#vawo#Q*Z@XOBp@)vEj*(Tk|PHTq(+a`#VD!Q?>( zM9R^IkAYV79&3G8S2}F^$S@z}CVnry<3{jh75&5ZPamvi8ss(8d;DMjXDpJY(+R|r z_!72tgmj5)TcN6L!ON}}#!~P{Ym+@*hj%;G*bE!Nejtve{W^Hsq+BB|zMfSV=<@cC z+_A{uY~w|Q_D>0FEm|*(;y8IBek6fnegEt9%8f4U@zsvO5D7!qmH2x)r={axAC{XG zn6vk}CGKCjR`cY+YWugDx8GV`eZ3dcHG92HM6VsLaIL>8b@l7hTSNJl7qVa5sY!7= z>%eLD1lI}?B+S3h32l}2A^IH!V|yfw9&3ul zoIS39g3D$-S<}wRcn~q4+5OVY*r8{SGP2%1ZQk(eM?BH({`>K`klt3XGizOCe$^wn zCsutdH`DN=vS@|P7i}k9QT$VpFN?-GVRAt{Zq&TM?#6I*f2PoveXG(k=B!qfou>rLm&Y8v<1`-d#C z(d9iWAN8B&hZo5uRn?Rqd*5I_i2^*u4=m10D7Kg#_~)1>2if2QO;5 zw)zgo3>>Pmz128!*+e=-%_Q^Vi_3~}NJYEb&13a87j^uGZnum#QR?l2E9Xw~ zOC)dlylP#`u!O>zaukovRtp771 z`f8Z%U}oYg^?>Y^NQFs-3U~)@J<}%>?<2nI{bOTthlJ&A8-EH9np;S00dhJI&BZsmh>1@BDX%@A*+2GXp znCo0_F`z9f8DSMNa1@Cx2{KEwfvQ!nsNeo@w8|=B`RJ8j;h$TY8(TlODyv&j7-)!4 zr`SA(8q#PN7NA^I|EAWhk8woo)ozR0n-QxW0~*)baxOf0Kyi{>m}m$SKXAx}EBR@g zU5P{o`BfEaZ}^&FtCc^TQcR?^r;uxCuv5c*yT3i!c=PrWUcxsxw3BPM(SZN^PC`nu zGU94TSVQ!~c5B&p56U0ot##LXB&2%-EM(CV+=N?_hTw9~`2pwCslA3CX zGGyuRv;M0#xNDTxsCCgf$5>u2l-zkBwF?UA`JQoXk#X8EYw6ZgH$l!@)C^~18i zHRha(7b8EPlwCYEcV){FL~1b2^A#6;e`axq5VJKJ@;QB=+4$m&6%mo8IcCce^WQj`#6i|B*-&$IRm_0%2@UXaSk(0Mr|*)=q!2`gO$Mjjbunm*sKL#1hBk zr>nJz`x$Dnvqrw1|5|@6P^$j+o>P2=l<5`x>#RY8&l!Xd+Pg&l4VV!U$JorMxsM^6 zg<>UCTqCsiNEi+E)9}X8*1=fih z>@*$el$=JXfJqmpR#guEmMOk(VU-u@A}t;io0T3Q8uL*iI54I2y2;k}UDQ}f5eupl ze31)6!gKttE|2uq7Q1}*ncFs*!mG97a8I?yK@Wc_oca}0$a)xh@dWazLc`7G{8t)^ zMwNKI%`}-WLGY6j0I3aiiS+Znqq`2kUk(5gQoVX36pxxbzf!Un;5q;?$S6X^Y7+ox z8Bg}&yyW{T!pWqlI;&EojdwTZiYEhNl2M!MJ(C7Y+#Mnseny)2`BvN2N5?3R>|9?% z5v6GOyt{yi_2!kNooj1-X);)D`z|LtDQHw_k^$pohMcT64Tb%cmH-NS=A>`RWa-a; zvXj*&VABQ`1AnwA`}A#eF61t#LO>N*xIk=G2Bv@BA8nGFP|I2g6g%0x-}Uo)p83j~ zdABpp*CJX)ko0Fpx?|Um1l(3;`x0^98wPx_>iFP_W6_Phj(epL1nsq zHS(rZAaP8qnZ$==XP%ZondRQ-^vI`F*QI*Pj-XA#SJM>ix9BXZDQ@#uKRvZlxSK(p z8_<(&sn@PwfDL2rFdcf-mDb_5Ybi+g{1#f%fNhBf57Ww5mmkLPGs{I{&r38Clp8-N#{w`liI#UGxTRfV~?j{ZCRU58n2otafpn)+qD zc_p_f5F236pLw34%eUG`iWq;S;{v=-uwowaGEe<(*@3qF94dC8?;%=7cVWN^!N{>2 zVNL4P^o2gTeQ;25l6wkop|+M;zjnNyg#O;m#doyXpHPp1 z%x9butaP~<{v@RW3+1j24-(NR>QY_4qgI$cdb#Y*NpNl~pyGC^g)tV|>lRS#lzOjG zyTv)%69NxZK;&H1UYz@erPgW4oT?OJ6eR_R#*rB>Z1##R! z@g(h*9OY_%?jw7k>Hy@>%~(uQ$pkd1!(bR8~vcCc)0fBlCR+jGvG_A>7u!5fx7(6GRJ zrJqC0y!q&p%i=e-3eGf0cFK&aeDTQB4u_oJ75KvU7v)Xl-SS@!J-V_lb=qCJli+99 zj~-*%SqkpsWF!jSoJ&7EZd@`#~Iki!~W?7;qD}?^n;3<;-6pI zYe*MSR0FhAa9kb;NdkBCurxdXR!i7nr}s@n)!ZdndYEe_?|c2r08K;vpn1Ax!Pe*r zd}BP42U_D}`qn6~DcEVQn{6qjs3%DxUb2;ne#^zo5HU&1sD$6R6h7Ka0DeNu>Sc$@ zro*~vn734P13^TBU_QQv>lG3)H~5foE~ZTo9@S}^z8@FM@!K-Mu5vJIG*ne+py3*7 z4S<{GLGRIHu^q4}V*DxxS={6@EkH?dko!EbY5a(&F!;CMrxgXLX$o|<4&cg1e;{I~ z31~A4P?JNr&%p=@@DD`X4}tfU{n$ntVw#6t6QB$QVUk41UafdRnE!KkU~5PqT{?KH z4p7L){KWgs!co{usMi!Alb^r9L6W-w_H0xQ9M{K#BG*zT2xw0l#$W>N%eR&!iI{~Z z2)WtX5I^o_wvn~S`n|$2ckE4C%(bhSCt111q40QTOd};@gap16jNYu%JIl`>TPQsD z3V27D0^$O4OJ1spZ6&Vq(J4gpGBt0(ExWi*jDUhX^854%7uL(e6oqBoB1)^%FdP!R zmy2PifKG=)5(H{nqEts0KjY#K>0+h zy#P8z%^zXYgEYZU1liM6za9&)_#jqDgFe_#y@)t+)e^l*NY@I^>=7D5o&a(u1klef z_Y{&ULfM#~Wte;h)67FTpXYm)f*C8taPbAYg1if-FLnd2ti@m1w;y{Cf6-eRe#0cb zo#tu07Hu5Wk-s%D;qN+fEvQvF3zlt8D;h$4rqO8xcs>X6r2)_gz?D(3PpI%hdJVO!77K-4>jP`M14sDi*95o) zAHL2Q~B`Uzu@mpAbYtrPw}*0ZmdQkmP>`#udx;R z=ngG7c`$I5507~bUsyxA5+U{cv+LHt0XF9H8m^HGE2Ba3e&eQhrf53kDLV(qS4(yF z=}CYkli+XHklRan<8<@|&ebPt&>M6##Rv43g1*6p`caTB+DFW|nD%~=7z%oT9r_E5 z94EmiILHhhJcw{4L%4>V0U!*{z#PQ9ZLb0b`8(Tj#41IYp|sfeR=rSe`*0Zi66yLD zh(5?cdU0B&3F(f5orXNfq7Rfh*YeiZ{@2Iq(GD3EN~|UYGfjXS(x8j2l`GDj`Lr{I z#E@M5`}cSZ&9&HEAas_E3ME1sDcH+2cEJ#~CNZXuhAD`J@DD-k3GivoZG;%?&H%(r zh`-awA=pc4ingYD3}pUWjMVA{t#cpV;{cuz(Cz*1YC4t)trV?oV^t2|DAF*(tzMH? zdV>u#Zy+er(Y+i%fG2X%6J$WWsV)G!Q~)CZ*bDpu)u2ok0OuiRu>&SrC*fVu3IKu|H-)rz}>GUoCV-Uh5;ac*A)B0)?#zB z7TBUHVr|Pnw;x-2gK;8Jj*1i-JJxFw-Bzy^F|U^%X9 zvH1r8bB(B*Es04;!&H30@us-^gI)qWNy)=$ym?xm8euZkueC$Bb?v4jC|-5 zmV5^BlMdHb-&vK;f{M-@j*62IRGZq^=hp%*t(`X(_EmmZLnSlDs)JpPf5CrHt|*u| zNdRDL+XlM4Zm=?Q=JMV_T4#ya9TTqIr(T$`27pFuoHY}9Jmiqy@$zvWfKKgAePF$t z*4#?LTJU`K8QTw(JUw#fDeXsLG#5h>w^w)#(4=7oh|f)~0@NQOo{)hT02ns)*fDPq ziHG43hiBGc^Gv_$H(kqHDL*Bc8_1(*It-}j^qL;4NCK||P%A*$U~c=j&GgHF7rwqn zk(>svGx6H$0khnQu$RCu$tV3EAGggA0Uww~67MDZx77B6Rt0DLej2Q>JR@Lkx`1m} z{GDbjM;`_U%7eUgd9Z*l%>x5|Xex4D$k$@)V`l|Wz>-yR*V&r@DHTqNro-S4Wo*R_ zS4}DJ^cos^tmJTAuj!thAR=##ilj->8veq6T?KGywXK);#=2l%)1mw?;Trd*yArP6 zAUa)*o0LE!kpl4e8dR3o@t6V0AqZ)9LjCwSX`9-RfUrX0U(mLv&+<>R8{tVm_T#^1Jwb@QDJ0N_7M~Onn2j8a zA{#YDay>Om`v;zg6_<*pVNMHZGh{)|X6n_w<$-UWHy!Qlf*M9iGya3nr# zs9o<#bEUJK=i}cLU>3YBbE@$?d*e@b;K9b900Agg;DYu}Xl+=~zAX%M+IQ(&AQI<< zl}%lHf`Z^qErj*4nZN%ePE(R*`S}mDWle0`hBZ8*;MdZ!oGS81R=*ee@^4pazjy1s z)~}>Fi@^zm*8yQ`IS+#2#Z7_&WP5?t&b~5pYk$oe0hJ6y{b0*KnAp?DjS$i>CJw*_ zKzV~7mU;P`FdyGY!TAzi4fg?jdF%x?nknEsc;T{3O~mCB?KB1{WKU-9$-X1j72l_V zA$^D>O24jgkrjW@OWiL^IiIn^>nGc@d6=aF&t4x-hu=P z;3k_>H^j1U{YxU8fzM}=Kp=N#$RSsj@$4FL3#CundJQNNUgdP#z)f7%5nKiJx8IB z?ilS$c@vf;t`P1*<#VfUlswe056)r%(<`(nfZVv-q8TW~d1}_hZJjWU%UfDRNR9TQ z9D^ac*4a>*%c6bSr7&5va1=0hC|&Uw;A#wVPA&tYw1;orA_B)@PDAyS-v33Aud6zPZ6At?;sETB{%aMamL+tJ+b*<-l>kT*1?N4wH|kpKLYIX?R2z6`5-lU?6# zmKATXkGwkHKy5mP)(|k*#$Si>4WBQrSps>@TJf7 zgi9$2u7E~slRJ(KWxZXF1P=sH8gNwjHPmo~u2TpleuB$4q`*DFyVH^p;zU;zpR ze_>MV98f9RsPZB{2G{W6RXR2f1kF>?l<0&UYNqb1r1k7;yDW;i)LXp02~y&Sd0D2| z)H~l<0+&@fAJbV=Q2;}X4v#Bj@{J$NR1KZjutlH}K!ws^g4>VqW3u5SX`NE5{IeE5 zyN=ye+7NJ#v=SjG^h>m|@1oqa&1lGCN9S-9il>9FTed5;*Kt6v|rEEPB;L(^f3T` zGZ7&Bgyn=dNp6O|LP#i<6Vwby_nyp**lJE)mS0R%E2sh}saB#;|JFgVu@=C>@+{*g zuNV;tALwW}K>8Oc{ZC84xv=_S!=n8rrdj~|349tzhh~^kLIx~ZfJG0|luhij_Gjl< zCsB&?A_o@2erP?rMdQO&@DMFwr{DQM%yj3dH0Y6f1ce?d<_Zlyn?}Ml=3P`XWPO(Y zNu`{t8^d_(bIvR)rzxKx>JSkk`K5A26dfYytx zW~7se@nHBMGBB7x&``yN(~ino69f`u?UkLS?OWVWR|wF zMDLHYkz6frv!b0aYZq4Gd4ou7IZWF^9hI4_VP|mVXjX%KyO5+<_L9oh*4~pfCqAbtCKe z(^n2m6SYIWJ_A*X3d2}#Vqhh?mkfY!okCQ(jd9t3>IU3E7S|3T@^Ro0he+oQPCI4! zQLqO+l9hGok>hn0q}c@57x9hp)n8JVPSrG^Cpw?GzH>p|r@kSa-KnNNpn=RJrX9X3 zj2r5(Ml0Y(`HGw_#jk)YgH{kO)kNf+fUjWw{FzJ|U1S%y5pZ_U41bf=C+74T2oe`T z8E#q|7n5%}cdM2fIZNQbHTDe=doUl?GRfMc^jP0?oJD7v{`0VdP+qOlbJHBx7+~MF zVW0x+Dgf&S$xL8fDmx~vXBqM0r7FknzFGcx(KY3vB9uy)m+yvt z!|n)kbxv5LNDtA<@4$nyC4{K4y?iY0J=1;SAv#&DYjSo67J zX)16}FN zaS{xO1XG37(;Az9 zxWwO7tfJc_qYEO(o0EG@RHQ`fV?9mhMK5*ss5Sz2ne0f9>+wbMcu{#|RjvGlm+;)a zMi%_yJV|D1^0&m$CcOB-vfP^;+1`AtOBSF`;k5vcB7xlWKAOqrFRzkMObIRy1e{Yd z6PYs}&pWV#^ICERh;GthrcwQ3LA>-+N(NY!%{$v{kqnLjx|+n2`XCJbvl)L#b1f0j4?5G+ScKfVn8iZ35I4IUAsg?M$^Bkx*; zF)vWF_OWmp^ruo)S%+yN<|&8u17ZFt84kpLrB}{el4%;5WV+Ch-uKf3CZV)2rVASgT5b#Hf*}HiaO!hf_-4AUbJ?Fy zbaC0NI4xK%7w%7?i}IPV^el!9JeQ{>5|{b=K16|kx3eB#F#t`XXSr0Q0eGd4>e$1b zdz->lGN&(xi?Xe?vYhcqLobz6T3I3GWw$7Ji!kLnKGYjuE+n*WQ2_(D;I4V;nkZ}< z-Qwsg=8GZmgrfQEyy z3T#DJEtslSwQ(JwdN+*G$&9nd&8!!Xk(s9iP+@KcGkj-_XER{a;CqwILln9WcMsC* zq6^_+I2=t7l!~RKzZx1i0RZjvw;88Adsv$-u9X>1f<}{=-Pek1D@9erm@F1fr>pdW z77SOB7ABL$%n|X{%KFRM;dii7HZ#35OpEAQi(;incIKI#p>iAEJ(tFyTO6;-iYhPK zhi69MnI?o9tMYPfc4idsSwvXIEWqqUYBm$M>F()Zz6 zF^tUjoy=&eO)fFhwM!(I4_*J06}#prwG7o1$BzwU`q`SPyXBZzaI~wOw%eTkdoVc;O*T-ctF5;Ojc(0tqR9mV**c;*e+Mpm84_kY3}^>9NJ5&n zmqnkx4rgD(nCzI|zUpBadEgitLfxTs-|6RMFC*+`c+yq@0FMJoV?3Y&@otHU-M?3u zf|dHu9Gon$h>gPk_Cmg6vqXs?(K{3$7ni+*HMK;O6A?S&R~D0}ySodh>}9b$R4siS z*EpqnmCd>!lO{wum(2`HYkV}3!>1{-iyA*`fz1#PwHRXbJ@1E9e=k32@Ve;9ciowk zj}O{cfZi3j5yERGm7z^+-fLIc zXokEGz&AV-e{KTQ7-;7)dhb(Gb`-~|)Xng5re~bV^%MvI>_70d zxU1kZ@zDzKL72O$ax3&o{-d!-W;7oBP;dv)P_AOH8Z<9?^YlJPU_~~){mkFO9x?np z$bEoklu(I~{Pg-4MK=p|ypeIYHO|DNOShIVzT)ida9ynQ&iIV2>Wr3q!uQwrCE4~y zA2BCfr*P!UE}2eSA4{HuandKpyD_(ST&vz;@}W9`>l$5PQ~YDDB0Wjep|z4p~ZF`(}>``T#@dD&|{-k?@I~~ zN!r!Zop0JUbz&5zl#BXFu32-P{XHtX?FK5kRT7Z9>Uk2T*W0+K3J2(8n!_F?BbbuI zz>6+)lFFr9TvzU47J&Zx4})@15hPSj@v@TMlx|1y*5vyNfnwHw9=E43 zDgng~7`wIVmSH)+ndRK{N!^E>0KEw&b+0s^mdQY)VZH0g5*4gL?+~qQ!9Bu@16Fe$#h@R$6I2@ zH;g2P;>(ur=jr^xDO*i_JmuOr;=lK+Kj&fF-=4a6_s7l25@++!dmJ?3?>t-bpXqEl zc?4V1MlNr9&F@iEME}aGRcUuvklcT^a5Rx({5K$=U{vu4HOMg}_{3rxdtS9Ahx98T z=kVh2!4M~@@p)GI$pPr8%}fKL@mY)MhozH$Yc*;g75}X@wO48i*@Ha7oa<1OuU6NS zHueoX;hnM39s1R;<4aIu*HZiw0ar5M`}a{yj{joARx#VZ}kH@9` zgfz!VY+wr5=i2xMD24i^y0_<}&H1s*Z(fqNP9i+y3ii$0oG^Re?Twh7IsDZ<0efPv zqu|;OU@oM1*o5J8K198@`iPZ3WZ$FxU*`6>cbDE-j<_2ZQWN>0J@R3V(^c!xe|Uw+v4>TbvUHJPz|#rf70r)vWru2Yt;hqcz1z6&eMxybi5 z5s%umj~Y`t^-}ND_>og>%I`+{ax_YU56OQxm>4y5qVnt$w;S}}b5C-P-H8&u`iks% zSIHH8X$xzOJnVONqw&PuFD=U}@1moZ&POj7M2C1RRK7W7n`t)L7u8>o(^a1(1|Cfca!Lw5nG_O} z6nY^ktS~A3N>ap)q{yd9QKLzx-X}$WONtRD(QwJJyOQJfCdVI5PH;+2Jeiynlbn1Z zIi)c9^p)f@Hz6KKDNP{I}!_!elxw1>_OW*t=Pn!hb+Lwf}raj5>8tzUMxzEKB}+X=!uF z>U3nmmV@W#{7aFoM4jP(D-kteU+>0FM^kdI1hPYOHGHEHC+sixr|>mU+I`jFuX_;2 zuCf!Se-y0Lesj9;D;C_lonpMjd|xU4;!E{~l}P02*3j;SH{0z4Yi*m8JgaCf>2y>1 zSz^+5HSDzHjjbvS^-^IDFL<*Ff2Q`0Mr2aqrKYXo{*4a^jsDqK4=eq^bMovQOo26DR$$j6$qqwat2~MdpFSWdj36 z8WsvP8u)dXRGqVw{_q*oDQ7~37CPaG8zsc1t1&t~;*JmFx<#SxP?h!vVWaAJi(*y{ zWOY;j<3O>dYkSmv-7ZO8xj*7_^ZMyn*SduAu-Ek<$RS!v!-G;)lrG-VMX{M~RbMRc z`s zDdWs7=_2#4@h*I*sDBEtN+U6vKA=VpWFAvUvsv(f?Nl>gTUl6ibz6$)EBAztI!Nvc zBqj1Jp_q^NBrk~iN%)@NC8kt@Q0e{jR? zXOdBd-2oTQ*}7GRFO&wqc1@!yyPQ{i;jLCKKQ}gz8d$kG?s5n|A9VS^&o`9JrE@)L zPIysUpqJ2|p?cE7G6AZfYZAK1w0Y6jt>kLsQ+ix?P1N$lA4vEWTZ3PgAEO=i+!aN6 zmMF6b&mot58hLjrn#+6T&g$MA^m#*;J)lGR5T%$%{BqkdmZ&VUjyvf^J_lS8V>J7k_8rh{Q>eD)LoY zjN_MWaVsyI?nu>q1r@lgy2bAIb&_toiIDU!jDE$lhAQ%14I@HpvYGCOeogmx-9G$l zUcre8HGCR=PJ(PQrwo_g6W@^~4?g|8JY2*(L(DOyPv#3gB}hggEwDhd*s^`z z^YVFech8RM+=X}DU9q`TP6VxX)w#y!fa0l12`xVFRz8{xaFh!n2FoTL3wv{J;r_Lz ztj(^?Ivd?8eZ879Sc$#EhkV5#@67)$`p+U8tit(3M6Rf-vyOLJQVP3^_ApV?GJ{}7 zq9UO6gQY&O-6jo0vifSrvyV@=oA(U7?*FJ~jfj6C zD&i%m6|FjZ9dFJ=Xy#pER^3@G1EIM*f!X4Pvq$XiV1*LGw?E<=c^DQ^-xDHXSz0p} z9M^uf(zdae-EeM?#oqCE0T%nO%fP!a;<3{%w-@(p6T%a&xJbdxwYo*EGywWq=CGP> zh{ix@>`N6Es7DND!4=p^iaNsVNcf+YKwJ>x;`xwqcKF}`Cg01ZhOM=$3bJmuStK1J zR&1VjHON$cW?&Y)ei&p3KxN5zn$2r)rChF2{|lFVXJ+>Fh7lbbEkHZ+@L4jC+KxyM z!h57_tRigph)^d*L>nzdOja32#M-*^-8QwF-v`6-7j06xnpIFd50eIv8GewGt!T2Y zrp_a)>XZ|gWYaJz-qYER@oC~jQ5Mod1trvdX|#VvS;X^<&(7klU%pH;mz!P?zqom4 z^a~ZY{=tLH>by8L7E*XR)e{w7W(%PL@%xuQ8hYd5DwD;^w<7Jof_}6bs1(P$Zb;p6 zy9Lvrh6ZZqioRP!rS+p5SirM66cJh z$6aIT9@}>)Goy$#l|8VN;ea(9xO%nbtl;g;or)VqJ$^dr7ymuK_vF?`8swpyjE(0| zh83+?qb{)3?Bmw6Drk$(-7~JsCLY403^j{Duf)%)BVr=vG~ltIJpA>K7P{q@SsCa5 z?xyZGSCc6t8w+L9m9?aS~^5Hw~?=iZAtV316yT}SNRYzXNVHq?0V|4&TWWv%E zt!z|b-bRpPWbI2CibnH9l=NP)=Ktj?vpRa%=3)d$CbAR8bavQr~` zBdaFG_Q&N->WP=jT`m^Nf^hcB8s;VU~74~eUG(s4+6N|D~gXu?;N|iVzd@smxp2 zWRaAsmSpan@F3=a4b0;D&HtiMj?;4`1iXG(6JE_YWbyUr_im8sQ8T=`Anmo7!3VD_ zm+N^x5)c*-=E!sc9#sVaJxH(|QL^^U$A5ooy~#hxaq(0r@JMFsZsoxCLz@JLu*7;8*57vCdj_ut|oz;TzV~>Mt3Sz0Pd$ z#yAq%A-B*2Y0d|*DDw<>!0xW7y~f~*J^;_ByU+!FGwY76I?W;m#5WU9nQ>+v{^`FClZ=|I;8)HJ1lhoC_9OU$(1|PMLvX{6xqEs==x9f zm9P#dy`})$&jw;^3K&+-sx$=(3D1bnwRVutdr0eqC5Y5{%O|37NV^+^MPoQ9X2fKUd~e6jm(*Bm-3GIeu#v zSgQR-iBF0^_bGDuST_3SRhb|^@}h}oQpYi7N}*{Y00NZW2xHrj!7ZIa;0IBllAA5E z1E4~cx2QeV7 zK+9oAZg&DblLN`0qf!~U6MPt;N|{1Dbo=W@@e5VbZ!^EXO6G# zr9TVA1kr2p?`>00qN{jgBo?$WuYLBttbi`R3Y6O5J8SU})QRj409Z~8()M1Wgog+h z6=x9SH#3Ghy0(^KQ#1JcqR(wv5Ct?D5#IUayWdchn`dlmo8A>hEJu3hyUBrM-~!Kg9t zNQi7h(9R1=@_?fB6NhYv&A2?awveR2>vD7kBtb^V-5l4bfN!&6+ignM@Mx7F>7Bn% z<>MqRbp)|W_Cr`ce{9iB?NVn0EoK0|5oj{L*wN^?>MBrlPp03ickSq|Z7n(qkQQ~7 zB0|079VR;F{QhXvYx%k7C4Y87X6V(tF_F8MUQGGP8LL2w$Q+|FOq*>|5O1K$0wMr@ zD>@No+w@5v@`--D&?|ek!<9@TocXs9*Q}t~Rm%`KM6r7*_ro(g&Rk<=rB>!x@@vsHGjHSk?td#cTIWxG@v(An3b=4!I228!W#jZh@~l+DE4 z&l$u_zTfRB+YQYwIB*$ZZ#xCM%Vmn5>dT)RkzMUVXgzQE_%1Kb9t{q#>jcX07<+rF z_E_SuGN6!i=mCrA9cTo_A1rT83vQgBc9KKVOkyHe`$SqKsD(XT1rjhd019k_Z#xRR;-QZ`KW-1 z0}~AdVMq+i0zd$&4AjhWL!?ieN#IEm3f*xj*t<3mw~Eo(=ThkwS{P?7@;irOQS!D$ zmpW(HfQ|UMUneBA-i~XLiucHwa{;O)49_SPzp;|Rkr%gYx-YpDODsD?Y@o$?i0CTD z;Hlw!&d6|xfYj!)ER#L@?=9?bBOm+aq(1%kU;BE=k;f@u^z~hlq>Q>VmmtB`Rev); zJWpE+&k@nX-#lL9K5=w8f1PK0_quqJX|F*UV4H_!#4)Tz>D-$PWd+UiW~*JdabNCt zDI2Dp4YPPhz5HaZBJb}rWJ1|AHuG(9P2M^{$z)0bpbK1v?U~DlxB$NRDOk1D$gc`{ zXS7%;6xYv)X>=^AMZ$&LffgeuiXo<#s1#Up)oLk^$}9qv167v$JvuH~$)Mdh-kJ^2 zyMV8@ZM@k%Wt0Q}!At!@3+is(Wsd|0c9g}PHu$77!+PrFQ3^h-i?RlFjWXzNBKt`Y z%nH8(RJr1!L^Fs+#j61?rdG2!aAn-3huF31n5q)m{5&@*1d3)%^&JyYo5HS*4EKy8 zWjx>GNe`l!{|UmwOm5Iv8Kz`iQT=Y&pcjAbk3Ij?dWt7~XaN@H&j~npjMIVs{l#w> ztYR)9uG#T=v)ut^W2V5%v-#xACK1xPRW5C;)JfZo^I>mV6(W6mfV+iXur?r61t$hH z{qeb%Hx$6{wzm`oemhWPIhBtpe;3t*U-@fJk7^UO0riB?e=9`fc)N^BWn0-ZlzH!? zTl)q(^Jcw@ty8r=^@2kI(wlx@ETOwI0q|7<0hH;QowKz*kUR8q_izF|7SWpz-r*V9 zc6r~xhT&$AaNx}Qd#_}Yw0dOMfMkvX*^493Cwux@pb2KT@w|J+o=|?ne@PODb{M~@ zZ3H@u!0vO&I6g(1J*CuVmJtzAZSnRwL`J}Ho5sl4q@8_q^ELPQr=K30i9AIw2fq^d ze}iIaQNt7JcCh>KG3G6ky~7LIORlRxTncOBU(fEt-JcSLJm`feZLKsh zBblG8+X|($@p9Mn4R-@*vNu2ydie!QrnW10PrViW6k+wKACI9 zQk?ZP1E=Qi4iwO~qh9AnI9m%v#{Rg5of{SWt4qK*Xw40{61tsv&NB<*5^3cBuI7R0 z!0TUnmLBOTJU6@nIF=fKy=N~SBk?oxpLt<)?r#gv1mKSsS^#i>mbxuElDHd^p|d6E zQYJMyhS;9G%Fd}opI`Ka5dnTKVGNV21wjgT$$KG!CD`?HSn9;5Jcre$@M{qX@5}`1&31GXw1=nUh?hd+PaPR58Bm{|ue=TN7Ly#n*#T z8$B4h(cLi#*$ z;C{c)x#yfu1+ZM_3F%qqfG52(OW9Oe^zLD%joH*EueE&4tLXpoCs$|xt^tg`yzbNe z&?8Z46JZ%0UmWwXQm{?rlqHkn3%P*Cz9i8cadToE@bY|aA}Z@nI~@7H)t zb$C%LaFruG#F_b2xNwPwj63hg0~Mo$OPzdb+_!)3o-O;n?CXw-++Z_v3J`1gj(N20 z24`ejDh%@(!OVDQ+Z<}P_YbMonP+1*&*`S^BaD1`AT!EJxh;;B$~Yyn$#Nn3lm zlan~rtydEb-yx#6e7=A{ukR4%14JZqo_5)#?D=Q7Do{9BODH>{tHrWa#5=~pB)+M? z9Q^pcI{>SB*ZNCjO-Me zPxzzF(S#(-ROOtw*_Q)D1?#_E;#vBBtdipF8o$-b>HSe-0a@`Ii6nByrmYQQ8%Wo) z8~H>q%rQsno`N`tzWa6)<&xeZr{qrXUu?kqDs=BDzQgU>` zk{$Z#+MlN@W!9thCsEu>ttVG_O2VI?lY^iPK}Ajs%|K$~+uuln12Y&>Pr!cd=RBRj zh>o;Cg!)MN&VMQke~N4SxX$zX@UzaTumUHe4?UsaN}%XjL&mTbMpPiL!v_IOQ1~x_ z=i>Ra!|%9pPwuVY#;|~wT*Gky3uuRiTP#pG-ddpeaSf5Z9?xY4tB9KNy#k9qcIHr< z=i_*XAc<1{ry8GPU(R>8xZzhY^gt2)$wX;t|hh)-MUa z!i6_Lpc+6RL&ZJ=fGPx5B_J9|npSSx7gztV?wnx%WRCzL1x5zp$e4RH_;p^Me>(At z9V0`$+T4|8&M>ZGt?L=`bb$E`A`?-ooi&$hlBRi9GqHZB3&>l* zzvgN{P;NnvWPv>95VuUG_IPd$7;DI*#lfs)u$z*3CFIqccW!3TZ>V+tk%|P3H|erPpG8!-{VL{mJ!fupkPxP%|-Ngm;HhCtK~# zQUMg|<8icWAcH!YLl%~AC{2{(4mbObv-2l~q94{MLxkqu%=Hq_sIPaC2H_CCoBHDn z9-_S?1xo4*w=%1n+_fUKS9v=FrE;cN4+P|~429eOaM1{oXZSdeo<5n=7Epu8f|{}p znm|KYwu-ga(j4vdSFH5~a?B;?c1JPbWyn193cRP2&C%_G@!%uY3 zk00D>x_^80fy6F`1*zX^rHbw_0<9n~xjLVe-&a4%QG;v*(u=Lb zZ{t^vImaxFOUC7>w1&@dB2NxPKF95d@k{g8APq<77DWZ^s3uBFT%bZt(1s~J#qz}H+ z9Stk;EC1M7c~$IFb3;}6r52Bs|D144!O~Xf);Kiv76U*2}{lCgNZ_gUKi{j=s(sRX*yH!iPrk zw&i2Ylz&F>)TOVsk_mG_?Gkh4-o3}msrJo+7Y3ve(+SB)?k0bekM}giYB_lfW9K-p z8xeGV+bSNLP#I*4cwzk!#BdIJfB%)ZI#h*AvNlZS$)!x_4ZmcmKU!XlxA_vRob9x! z>hk;4Jt0@a9vK0MV{_U<{o)&>86h}GP&66K+bNCMY?X7Uq52-YmBDx(@wNF=9K88c zZJsuP8>swe??`2A*Tdq_0I*zp7Y=$qM7`50Gn%4FOiYDandDibS^naZmR4y;;zx)x zNovhW&MX>`@*lolKrA+ZPkRQjJ0Mgm-%kwC2Bd^G?qrC@6Cjd|W5R^Ad~nQ`swLrO zbhv+!od$<7BJp{RpL9jZ9Tqj7D&@>83U?4{Q);uagL8DWB$N@6Dhh>`lFVeI%|?jR z?kV;vWH{dn7^qgQM062+EF`wQ&P@mT+`P;zLhP)ow5=yUiQRazNEuTM%Sx6$DO-MR zd`xO!xyB-;gLVep7OtlNj;oDN`#Y#|ky7qr8s4GQXO|7QHh}QrZn~W=-F6auiBkk4 zERHb*Wrb}_yCO}g`Ktgw--#IyL2Tc$!n@;oeWMTD`;T6zb$$&xg(H5KFY=_L_In@+ zEu1WLR|o;$7npE?EN0*+%I-J)d5>j7goYD0jXEMGcx>d8kJ{dxd1_DzDWnRcMRU0` z6ZW=ZvjtCPD9{mGP8~CT^vym3d~T8<$RdF6Zdq}h^-pdd9CNBJSB>xx_~e+$_I9m1 zme7m@{3QO*4y(#lv6=HSF5^Flf#V~pm>=mkr%Sh6mnOx_eJ3AQ#8hqzd2LNFj_RM{$> zO2&%?U)|+5+@V4x7|iUGE%oM#DV*ZN%+nch9O|Qhx+xGs^1ax@7h*)@-YijSRUy?& zWlS2)7`6s<1~eZrNskBm3RcMEiL$Vl!p4<&BAw-xgDKFkol*-jBVFt@o%>NdI8SS4 zL1JqcX3$3sI@`>Z)i-r|g%x~!IrC~I z?g04j|FE^S)DLM!0(rKVNNT?RK(0m#Ty>`bXsWL4LG~A2$+;KU%e~?&n2R3Jc9z%A z{p=qxsZ_%v5HtX%(85mWQSIeU8GjSZB(@@Bm3@Kfi)HdUj?Eu~oV6RB&uiT$D;$vKeaXWljbhmkS^v~XZ4z@+n%1aE|R#M!C}gw6(~q9 z__b9p{gf8k;r(F!zHE=qm_|5vqBxv<9>#UE`5AAczqowEt=??3iM|j8LMU2C)&jv; zzhv4U^WhSg#oNx5>v|0zgzD;LWa;*2dFl1}UZv|$c6f&!e4OzjWPK|Ng$80+>RvIj@63dHw~w*Z zQ+_UvD;S|td=jBUqND4`WEM|eRKD2XBgyCb;>cxYGMuFR2vvPIp^Rr_Tf!1lQOZ0f zRs1Z4BR!ojW)Ye0y-_p5dZO_#!J4yTB*UC{{feNvw1{wi&FLB-vld4|@lsPJBF1>q zcOHsU?g~EKCL%NXQa0VpOEPKh-o{Rpmkkd~hhL4Fy37>Jk$0+|@3qK|6&fDP$OV2% zd%v3ZBvX85NA|Pexqlu0iR0$G(}agTktdruzf0aliLJk>C-ARs>>Z=heCdBcX`o3g z@8CNn(Hv!StKA#hvPW`kACrfDl8>(uyZoXlb{cNAO5t+7ZrU@c0-+f-6A~>qka(`n zgW3);tEn9pvXP+6pVIohPRemYrT+S#$WJoyA)IUe9j{3cLUL;c9XvyrmS?5uhkSdl zhLC5$>d!oqbWoIiEF;K(?qCZ(JyF|*@H(V{3UfGgYf&uvzkCqYc8{w}g z-b|&6WRB*jKOO!j`=Z@tp%Y}9@-Ap^HJNMfv4%U{N~|L<#os}*O|~U4a2n1{*%(LK z5&EnFvkKr2F60jicPWKeG03IE1}Ta|cC}YY^Zh|mMb4-@95l)vZa^wb1x6fGLn9xp zCUdxuB*SNEwLGEhc#y!C$MPAD&&r!E`Uo|Kfh5T>NdH%^ySZosmh5*hALY452W6>U z=F%fL8Lw!F4f(d=&ry5>#5&qqTZL~wua*sZgmC2OW418z{`kW-UXODBr~__cdBq4! zf`fpFgNifwd*5?FcLcUy5iw`(+hZ*iVy2=j>p3n<_cKw`Po5i3{{!d*w;=c!ndcPg z-28H2I-GA+L5AJ|NM|{|rnQP8`Mm#=xnnP|PG>EehW0U#rSxq)=S@S6=GXR$P5Fwg zyIzX^*9OuQ%ilW&o4a0x{*`>HJdF97zB9=xOYT)Bx;J zD4&|Zp2mqBhZa*v^M)(S$;es@b}$!%d7;LaiZ1&`;RrJFq%V}_sgmNXV(YbHrSsVC z*nqiQoIK$fAU)hQ{-gsmmt2*7UbGYXEf@F|Gc$!bJe59w28M}Ee>af78 z&dp{024SenFv%?@k-Yp zhy4{$pu2_EW%>Nu$Jzb}%o7f?2|JM<;o4o~d8^s8Gtm1~g4N!BS#F>)TdvTt`(-@q zhlGp}8SFuZmt3R6&Jm0BKgxflNV;YeMe}YT21f8o?@bio`08C;+RKQf{c}Z)F&{I0 zA3lyc>F%!9V@72f7%c={?}_6pP)&7@t9(zcF7%DkI&ZX~`$sqM!VkUA_adf?BR-zD zBI?A=-qUSOh`?CHhoxC~^j=W7V0I!i;*%W7B?VaO5Bb$~_M*1a14+Bsx1>c=rEl)O z4kFE$Yr$tO1S|wa{w`Ma5{!KPExIPbhTGCgAi?LuGlh%YqwJ=&b4{%k(AOmQ7)|HO4*f4S!uk zcJWN*{F%%pn%Mf@vFg2VGzNBiAw(r|}VDn4t6H67((UtRFl?!E+D@&Dc&{c1}sy>ue zeO{{iimu-Cs{UD4{b#9~ji!UW=^wmuMO?~Obd%xUH6rCT;>$J1#cHK4RSvgbWEUhg zUA(5@QL9&8XRusnJn#h}TsqvIs#fARjf7pVibh2LI@92M7h2LjA?k8_Amy8WS@*&1*@5;t^Um~zi)wxK zH}+ zJB|+du8{cOs&ZgBR!9KTIm;*6Na1RyaQZ&Dz9KcUnRIjIqGYK*g56GmZQmNbQoq?*)a$ zA(4Bl^`#!&w<1a@oaBljQ_oC|lz}S?gVgh#sHEo5&0(&kl1vujKvR;AsVl)eV&f2i zZw+!Lp)i|5FA6M$F`mlUS1jZ($4pdfineMF$ViU#VMoEc#;FvZX#!V;b&}ezlyCAX z;4Zq4X43)Y?9aX-8-dX^3il)NxVtQQe8uE~{j>gZSSf{jmB4X0=H@`a6*oG($b$|_ z+Q+#`dICnv=%*hEE`0G_aI;H$@!64KVP-$k{x@WD@6R@y^*d0i)^1KWGKC?(Alg5a znDVt2%)OTEw#nVjqTgNdzgXVYYK<6ra9u*Oqp&1$8f$rVvcCV|MT6mqk1q2pCf9WxbQNoTC7***W79Gv@mmyY5&pN_duFeT!}yQ-jtjR z_mEt^>$1w?j?TQxV!17{c%64tN6)n<9k1rV&dvXYaVUQ@;>`p3B8fQ&7yEO z?m<40WhV$+?f)b@C|vGFy5Ha1{0x0Jd!y#Jh`b^u`MKCBvofx6`s^>!4LQHm)N1kx zObususq9dYcl5!;R@+q#(4(Mt9tsm%kCdPxTNZaJrUBpoEo3h zvJLo0T4Wu8*`oTe1*O-18IaA2HX*iY^g>b-<(Rdn5kGL_4MLbLFKW10!x~DF;19(g zsMQkiPLhg!p1B|}jYr0LCtUsQlN>Gw`OB6yB=&eJLfV|$OS0BFn^Q#B)^a@{tr)Bz zPm%95VIttT^W)e#mOWHY2^m61HczF>gd=knhvi-s^E)~B{rZX!FFK*IK{gFue{oYM zPxuPg&*Lb(r)c1?Dea@6p0alrLoz%I1oLN&k}^}DW!{xSb-f#ZXKh-K?s`v}1Sr2s zwm8#%*o(RkXyzC1iQdv;Jg%^C5_#sh5gt-+d@XJC*Vmo#5^~%eMN1|SRuAoDWY$p3 zICI#F^34ni5jyR1JM`3WVUS(*#Drq=Wi1(!dRSmrU(3!1b76_D6~BsaVTZh7F&!#A zyL&^pusO}k@_A=aW(%v{WDN!r9d^jxoA-@Saea*5c6$dCiM475pj=!ZL(upyuf&~s zB+MBWch&_>?cSXDe$z+KF*q>jVS+0##xKOo4aOrq>XtpCvuRG&=kaiimFM~J3UM-Z zG|x8jTWFwjvC60)5z_sAHe!9&y};V@H_4ymrYZW?S;sGx`Bseed z(^RP*AZHJ+P$ZZ3>I-G9$UXc)NGJ5)Tj5*r#>8*K02>rpfX$mabD+#YWr6Mw2g&Qq&RG@>8IsK)X zqDhYG663ZNv;Un%{`RA)@%jYzquuPahMhRJZSiF+B?Ui$qtTOzj1C zvdmLFJIL}}{@BZR;>leE5;~@yONE5@uW7&iP(r%U(Iaa0h$+e*`# zad6>qlV8lLQ#2-oMX7zccopN}3|}Wd?(8|U<^lID?3^caCZlG%FP}#SxZ`7Ap?b$^ z|3-Dlo8i#w2!8gk8$1d9K*c0A;Al!}hYvITw#;Mli2C=4#)>)%C0A&3+bbds$y!gbP0kgdg+e4x8Z}tpYwyD-LohAtJU`OTL`l zJefM_l#&7y>8w*)azOjzw1lUNQQ$%()fW>pg3P$kVZ@;FmMYa5 zv6MgR0C;$xS8djAvI65vcv$oJRqlhEu=|ixYz9cgdshj;B3uZ~7cM~LQzgh$?tZ0_ z9%T=la39XZ1Iv*&dLi9_Oyx)nKTf(d0ubvXa>!we#mC8@l`U>wKoBTIa;_84AsKSv zfb9vp+6z7z6JUCxrdVjrQDj9ro%^i6=|KDLXg&7_gcFMi`uap9X9p1wl95UH^ubtp z6~O&D^0^1hQ~9REra>43G;<@JR?Q-+d!F!yH-JPYXP;rYX9AMET|u&Ru2W^M*&Ev# zs*;osc_ECFv_>+hvV*}9LN`U@g;mhF(fH_6kU%@DP2f&Cmr&FV7hfdY(JHR|#hpf- zpbr+Rn1b|RORgBrklLmMhMT+>P@vEfEXIMelPJAG?GvhRt-wWT3X1aBt3z2F7DZ?K zn4IB`t@(Bl<{Kj>fHGuy(3R9kk3O14Y6nep^gYC&LqsG^9fBFfO08`$dD)&FV0=Y@ zhRHW6lsyuxxox44a;vu5b4T{ZC_Tu|^)y(XplqNsntG$c1YJva3SLgvW@8!19WpIh zoucq9%K_R)cl^5pQ_@}pO3c0$U>5eMo0c5Vl;uq;&!l`XHx zkZHoq0u%vPAARf{l_cI(cN#ZeN-E`NTZ=;nls`)0>BB-BGwgvow(Go6D#IiHu`Cf! z<5fcvugl-ft^ERO2ma8MU(Xto8@B5LYv-17q%*409otPv0HsrD{KX+cH#lr5K3u(* zl80Et)5d7Y>Z&ve&=#XaBZK;l<=EBB%sba7Tc@}QEIKn^mN(9<$n}^1K`suGV zDxn^5O3%}+Mjn}}h5dcOznv$jY|vbJ^sCZX*E5er-w|Jxh>malAk1M~gaO7})$|FsY6Cmm zYkcsA=UmiPpXd0DEp%xIZ6rENh4wNN@kemExMEV_zZVJIh@Yrh63`oZTmGg#ML1hM zfDb*WqIJET!l9#byAS@jWsKIx{)?ZyuiGy(QmAx64gp6sNFnQ5yf!M>!-x z=|@o*ZHB2C{~YDPDgo#NqqX(^jbHG;;V^?vc(A&gZ}@f{aYND?8u)V=TsQfcpbM3qSZ9sozU7CT(RQR%izxC=p8k9A3};mTS6|HELfuDoUNSEab-fo`hO-@!Hog85ZBY{%O}L!v0W#3G z|MM1BlIjjQ;Bz9w4K(bPC~#A{Npg4U>2FF*W{ep~n-wD9*lBQj)`1!ew>@f+GX?sq z(?&W^Cg#d*_TorL4yzChngB4EiNt9*e3}Oi=s?@*gno_6r4*cZ(F{4lvX@(=CpAcQ zB9)_}WMgANX8~6X7mbW@ zfJZPnPM~8n|GDQJ*Jl$2TxR6gEMRX&6;8$o+cJa-T*DQd(dD&*$A@)fW27aAW!n~X z(dwS-orNTX}k>1GSAoHgA@t~%Mp-3Xc0tiUn9Lz*ZofL_%6h9$wV~hKH!KM z{uYFoxm4MmOJD`4R`rOG8{C9-Kt?4%Bqu9|p7+37K?CAJ0rXzHS6{3g-*-A#Vyq z;vo%|K;`eq3q&NW)5r;Mk$XIzhX4oxa46;h^~?(|o`>DWW8?@iBqq8}0fTS^6$J~B z(vL3T8Nw%?%vtTqgow{N-Ech)TO?)tG-{+lO|fPMWOK?d>IV^DDQ8f$dd7JeJ9M<> zN`>~dHW0RVT(mx2tgg|U6(pH7tI7>Z`!cI+d{2$HIBVyLs6;3-gr;nnY?|`c%nC1J zk|A~+gmUY5S}7J;8PT>S8L&guG>(A0mVlN*laD_uhvHE@?tWLgto)xy57l^EfH9oC z)kvp|J3v0w)68*9%1Mu=WUYw$4Khk*wj=*p=OKe*R+_&X?FT4jT5Y|Du}>B&eBhX zYF^W4l2&Tv^1qY7FGNoLs$NYrEdiQ(QcP`YwH&+ka%L~pONp!ng`XtGc1v;yZ4@h# zDn8AcNQGQIH>-5KS>5?QPIsN($5i>PBuDZ6t8*y3a7Djcqv8+V3EdH^?&&-|vjota zEc>^c%4UK5g-adzg^F;e9OYs=H%MW)0>?b<o1m0`F7jVGK|gy0W*^7!gJGeR zQ**eXIn8U&8l;8PJaY0iO2ho zxkn3qS-1G-Eoii>ZZVllGP%m9X|)-a#Emz$bqe3Nf|z!IUkVX4?_5z3l)Ex_T+`Ks zTOq`msG@sMN-{IRb{5E)3{@Gvuk@4FY_5$Axcp+zS(}avoKuoZ?XKLiS7vEO=c*TJ zOa!HZNrZ`ZF3p($Ll{-{MA`!$Iy)@wJKrifkrD20~`&A^_IJ%NxJC=G-t}X zs(llW{w@%vVVo5QyYaF?vP1y+zH@(`hBE;DO}S{+b?SPpEO}QwjB@(KedSXDGBZt{ zTbl@m-N_8RXi}>*N`sWF?dkh% ztQ*xU%>QsidpxGuSmz2-n_ey{iGj`fWL9SJoOs|nQ~2yH<|=7`C3QS)+iiE&9`A~7 zE0MRkuG=cvW=dvb^mIZrgdev4xnJolEnkdT)liLA9AMlxD=F6roO| zC(ZhLv*6mCZY=eDTockEvt;(=$=otjN(nHC1=IJL;rY89FLD2@Ke9a*OmWoq%o;fV zL|4R<9enT`X)`tz;Vu2)g(@eGG=75ru@lI|Us0z48E9tHEKrx#sE-|@s^>X~2CXm+Q?YKN5GcQX;^R0fZ33qrb z-oJ=>38tGF_ya0gK;@U`9eMbb`lsG~xc z6q!Fq{S`3&1yF_`aBj>>!~SyJr4{3AE1PA;h>q;)d?6XS^9~QogrqMihMG84T)4$v zE)+>VNva}tDcO#aZQi^2NkVFhq6YG7y(wm^HAZQQt68&oZy&Y|Cb?@qhNh8#O8&78 zVsLFoNesA;KLDdf7lS_O*bMwSa`9*Q!kQBj{##O@K}HfDpawTaR|FWI-2Ys!FC{vz z>!R#QTt?a+_4Lk|n5D~b#&*5f{ZFD<_;!`qF3!W|OP$4AWmOgLk<=>xEz+_JQj<%! z>ot+9GYyJ{>hq(u*6%b_p`0La0|I~KjWw~W$i96;a#YsM#!=!sn;P4%- zKjT-#TV9k6=p`9Sb*bRY?{&C+%`wSSbxl>pn@DF#CDs$!jhspmO2@b1Y?U7K69A`; z(=Y_js_n-aIti_chHWtBbD2Y#LJGe1iFu$rEeSze=_8_EmAnA^@7p=2VPly2Oou<+ zNs1Rw5=@L=e?OU-9Wt3wPu{Q{fk~|PDa=w#xbPJ?2ww~?E(Mv_c|(+Ghrat z91p_3eK&*AQn`(%LJ)3U2A<&yYei6eri+%bqDu zmSccgErA&w4AYD!)5*4~Img~^Ad2nzTOo{p-&&}QM0r|z(dr^inNdrorEXxVB!k-lfE^{gT}BYpBH z_$r!;Og2=_SfuLbrDbfThz8=G*AVr-pT_Fsxa-9v=o+Y=bbFph$Q)U;49z;kMHj27_|frqZa8xYJONENL5 z-(tXo;?GZ57$%S<$t&y0_~AAm97tSJI9~<2{!36D#HVBNED{SKjxvvIY!oTSO;zRp zdd>zV=FqRjx2R~;T{;ZoP~BnBpzNHj`t=+Im18wv-0F;E9rxu(H9z4l0WR~d95PQI z%)L(v*7hG$C|&-B^uhK|RmB{R>?ornrc@dLIo8TXQMp{%cL-9#ubISGz4nx+`&niF z$vn#UBPnjm=lwTa*C}g zV=nNyEX{8OzSzzv-}!VV{noJNXm##tfjzDns; zLbjzOVz++wL>A=Ynm~zp_Lf7Mklnm39WZCk$u=!P$ya8ovb7wR->@>8+*^`PP6c=Q z=f}&vBlv#sPyXELO~;eRoK3K_j74V0&SB8)SHDB_4eBTgmYY(_{v+Hfy&%boNKbF6 z8w2+2Af;xES*0=XMy;cTQ0}BNP)O7|!ruevBeb7naPgH{DKfo|8~_i?b##~af@y4H zLql?B%`XMdj8rJ4qXs~6&$@A5l?dL_Z&u_gGe}7Hd&DKBMY157#vyV0dWdC~GiXnD zWVPPLa!ei-n`K)b8<5ta41Rs=U9OmOE=a}c_4!V8^DVqk8yU+l@H)Bb-$A%c*KL=$ zX)@f<7k9^Yd^4gQ@?_p{s>I^rSHp;0mO*X+E~IqaZrdk~B7BS?aTfl@4oX2;xoH7udo&dYhMoKxE4KcT=vlJIGs1smau!r7d)075$*LLQu$ z(N1*ij|ZfdYC6>*w_39=&p&TCnQ7UQj2X>W-O6Vn`C<_VwW zct!(lTyw~_BwvCv!&y@jNgbt`>dD+1gKDf2UX`}sml1axHEw=Dx=?B5?EJT?mwsLi z`z0#;xw(xY+kWd)*Hlp==NhlqZ9TKVSn543S&`a|7FyNLUl?5X(LjaiqK;oNqiN}m zt`*$3KE_Ak)Vs`K32<>D^Rh?M%BjGgJs~8}rVd5n&&1?|kF`bF5Wb=B7sbjA95r$- zzgs(dtZSf>SDY#^h%HHaN(3B}%_%*U#=h-dRpiA+-h(ipNk6g!&+z*xlCUa4ynYg~ z$%>~&%_QJ=eq&ssbLHP@Fu+NigZby@7m}1shvN2GTe>-}=eCZ2bp}`Ol}kz#esLH1 z(}rs=;=P02?YZji^Y0V3tKRo5?^oB`q~dnMU+{$skAuYwX5>~o_cYj2TmPb@p9=#{ zNrxIs4{g6Zh+SuVgREV9Yh~;4z;WTtzn@JF_j%6As+}ICV-Nx!E;;cwx??7~TG`^o zaXliP8NbW8+`xnbe?7H`!67k%|6Zf^gNJIr`nd=7`CR**<2YW-GZy>}B5`F31e{xq zp~lDDd-6I2&;vCM1}e|+v}qsDWu2xy<6PuG#J!%ZIKq&%-kYTB`K^o;txRP6iC>5u z>D&Fbmj*ngGC0eOMgdhX&`mkcCgb<;yj~^^Q8MixO`iD-3;SxSU1q3S<;QD1Q0l;l zbrnw;pd{}+I1{8>X~58|7)}8XpHZU9dnVKL`9ZfCyFNXzj^exQT-3cKM$uv}|I>%S zrw`$fT!uWAgWp8+V}H($rrkGw%%n-|t=vqktag3&htImPQVio99i**p>Srdu6J9k( zPi0kY-auaMu$S+!Rb--1C)!L|d$`+RxMRvp$@RoDf0OJL&`9f!_W*g>EV&RiNaRo-&-GO^!swjqp zpm{$^KofJr`USDyYH+mUNf8(!dHIotfF(1|jN?$BW}EW@XL4>3iJ`atSTgp*n4I-t zO0STfMd{A>=`XdACiiq}0*SOpfzvLgQHT60hYt#AdT{pY*bl%6^?9wTApZjaUjO-3>f3(`FLnqe{Zqs=xsa>5K{0^g1yF~n-G zV(S6${V4Pc1I`9{gfEU_<0M6J?&>z2cuq9EK{EBq1gYY&b7VuU%oB@cQ$;>Di-~$Y zT)gp+g0o=ri4m*QRNIEuC#UWc^Md4-tS5g@n5pATISj2_6-%AP&1zQ7GPHGbKLq4&mqR7^#0i24XXl7!Ox5d<sb^O%Jb*s`gaMZ;S*t0PID9sU;DtqU+>aS$YwL+X~i znN{=*UpM0`2BeIwy*Q_@sgb!}dvZGZ(E!HrudhkmXrE)s)88@o1jH;8Zckqgmg!d> z`BmZQnlkx2Pf$5woSQ5nyK~%4|CZZsQJz;z2XY>d;9uM`efm`1FBpjP2jM*JKa9>B z$_Q*377QQqe+nLdIr!vDJdtVJF!9H-p8a&@tD(c{%26zCE@8XB`LlhxUgmyA0-*vDJFRP*p_$qq19wJlTj$0UImM2ZxyO~ZnJ&(zMr4g^CYBcxh z))V`EZ9nN3Cg4^>YAj{nIJ77bUEgqKO3R zJOv^UFHqKgM|V!Ny~%Bf66Dw}{ji1AO?>Ni2i3XbM!0EIklx0X*}W9S(^oa`)gD5$l`(u!xia!_;rEo? zs8NA&ncMUsq5g}{z^RuE=rfIV7Y(#(djYdhMmF2jv>E!M7Q=)|8ez_?wk37MC3xsX zEjm>?kBrTag{&@judR%&t-f4ad$;y_f9(y_uWxL{rD%->eG2%()O=~NDYN!b@Kn*? zSAw6H@>7=y2}>V!UT?pA{pH>3|JeJlze3;a2%Y{WdZD0o(MT?OT1aEpZ;Z+0}+1wpCjF7bZ-*!x$zr#x}4)ql>cG{uc4n634{>vy`h`ZpN+$F4zedR<4`@b2~~ zvtgY+zg{fu%vaoh&_G(eH}X$qooMD$Jhqg#cT(HsPOcSmrhpSxbu@i)WX=SL{LoEj z=($9{v2GiIy1we>>GnwRjxr!TWXpI=?5~@J2BUuHFxmo@%s1<8gO?<_Lw#y3&zpD2 zUxzEB#xM?-yB{mr=g5C|h8yL{OfLy7ZEmK%JwU76AQU-av}e`nHoiugq4otes@^vH;_HVgkUr$ybOYiu>(lK1&fPe*f! zKV_D$7QQz(-%9y+it@^XuaOZNg;z1RNafyi7c(3?>TXgDgFiGH586F_w#8%~C2-?^IOa44Wwmr*|jcydLT z#c8i{RlVb32H>53y8F@vpE?Fob>nxp@Y_D)050HR_sAdLC6mYdtsg&qH{a~13gM%q zi#S8KqwZI6tl|Km9mKt&q3x*FV3~;2_uJF%fkE#X>NV8zjJLk-MZ7Fp29qYT7~#4$ zO*vdRSdFOu+nb`YoqlX9DOE}r38_@k4G6UiFyyhLIuBnl50HKCw7;URP}9cb@8Nl2 zhzUJ)b+biy^LBCM{A10NYzCM}ruiD3mAlSa@7`M@)z#Uetn;foKlwjFaD>1QZ5L4e z>U6}En_rjc8H1z_?Vcq(GfJ-vq)&Jo93PAi7 zFMHj++dsQwyg9vt?#YLPyG9;m*A>G!l%|G>6>;7FGjyi$P<;;^o_*gige+s`)@7HELxET9^cl(Z{jqsuB{`LB7xAg<;3Vx0s zwcquLd)fo(WH3E}b0B;tw6yuax47Lr_U>a}S_8M`gL!IF$p<9#UB;h=<4({kuVs^y z$-b%f3w>@eDYUb*G9r+XpL@^UEs5Q7yS#$2AdVWrv1cUje&&)HFGJn7{yffB*EqW3 z+0x=mk~U51*Jx--=W#kG3fo)m&P-YT{%m!)=V8Xxqual1YGN(Psh|C7QKXu;r0ldq zKo4OGGJ@qg2*EfW^?uAU8-$?Xm_!}p@wLy$`#U!+(FRjM{S$c6RygPLS!v|(9WVyN z16z>pUwQ*t@c$fV`;CvTUns@1dM>SUlMtr0wiD}02ngcP6!6SAD6p@*+byC z2$Fd?B_lp$h5~ZZO=+Peb|Un3^67R}-;nq58M>kMjpM-GFyhQ-t~RjCwj0b z6d)xJwdMWDBybXbJVVL;iFPY-e{{Y-fq8doCMd1WZ7{9^cG{N=rOT%Z_dt!gk3z5? zPfJ&O$X{>`m*KYWtqxqTZ}}d+|Ho#>)U61J4Z(bmPKpZwbR+v9i~(|yPi^yM?cYjh zJ*MPx&)Y;mT3fsB0W^194Efn-&yhUyUW|`>g?17@n&&86K$+loKU{KfaTPB~d#~L% z|Kp}iO`P*l#+~8_Y$CI6%V6ZUE~Q%%)pn6JC~CsO!DUJ@5wxHnI80*N>DQ^HmYPL0 zjcb|;IyR7n-^m(gv{Tcp4`Grv_r$t3lB5&`NloLy{aZKhQlohQQ+NAd7=*=kpc_h( zC-*lU_*kAIuV|Qxlz9xPu+LZDi0J$!J6%zG9A zY|L}jNLP|Ua?zb}hWZJgynD^s)!kbYOy&miRBiGM&zXK4D&BEywDsK1&!bg44di=* z=Q4B6^#6R}1^^u<{q{CW+7kMU-gG$_6ulc{7(ihdeyMcF6~6Ohi99ciN9lQ-joyB$ zRagU(q9^P|20fHI84a_nJxe3^6R}lEvz}WiRA_eJ_U%!zZuEOgGN<+g{PFG4c%0UJ zbb@@?#~wLIxo{$$hf?>uTZ2+)IovNmN~Yj%Vw5XMZ;>(qombg!lx5$gPi**%UG(!G zNBH>es&x)dmm2Kl#JnvDus4Hj7K0L+V(k;u#bZb*W{ho$gIX9T!0wBh-F(Evg^Z$$ zO48bIJ?hLljwZ_P(&g9${5_p&P`vCRg=>ri!2`i^VMMS-Q?J8Cxu0h$FV^if`;fNj zc8z62RM}$3O1)zgLe2|kx8JrGtVS7zj86-&CLzHYoy|!mJk&W+arbc+cu!i6Ak+$AE3ar+Z z)8nx_lt)CWZMXnE9CiyKq(f62HW1qh@NOsb&%d!#-~z7PdE;}t7qUI1qo1zMRfE!+yM+$pq?Iovtspt0W{p0Ui% z&?}CUFrMnB<=D>>^K-_ZY9$Dnhv4*_;4J+ZW=y8D($%p{y}H0yXr_w~AWh!}1`G30 z7EvI$^J~ha6M2g;2OUYIbsAe|@b**UqH-uw$uCO!j^5SC;5(;eSA-c;jq;|w#mWNT z9~VH(F(;k6NYm+y=rKW>R!gZhY+ECtMSx~T250Hz5~$&eW+GC?w3h*mZIJ8bm8KWn zX%9r_=F|@9=6H3#)zuL)Gp9VB+z}e-W%=^)u8-j1Wtcq-BF`<`-j>fGpVuK zuW3U58;@?uW6swKevo}JOEK1O4PpEs737yusduN?07{;frt31G7E54E|V$J$io%EfEw^iX{ zk*0Q|Uj4B_e6I*dHB5z5hl6Fzg@GFSahi`gaD2CEpCqwMYfqr{7NM$N+185<%_88E zobtC@0MUN)qr~&#%Ea$t1j@BWrj2{d{^hOoiQErTaW1^Cy=m!2tI$Z>Qsw1QzGhYB ztn}ET0{jhsL-2Bsn@J%Mn(1+ren85;~JP;v9kX^HPsoH&8a7L=f}~M17;{Eq-sX{add49GVMZwk=O+axLlTVT3H4rvmhz9LolF~uD3utbitvM>9-JghA*hX4LJP1xWd+FREL6Fj|^Dh<#qI21Wo| zBcMr&D}VMFk?~9mGMlECW3Op@T!6x{Nxx{kiwp!s#*~^`ws;e4+YM&Kg^l;N4u(4& z(sMbqG+xqc)@XMqKu6^s(uF&VY7r#|bdymB2<$C6?d8@|+Fi{LHFnXj-85JEV8uW< zJFb>BlR;88QgGIgtpTbVk7c+V&XOu7ON;*PM(tAUCCEx2)M*qedW&YQT&KW>Dp^qD z`7EII!j}Dp09u3N6Dm#K-f`MgVwA_x<_AliKHjv<>Hq#*&+Fs|=7Lr|cZ zyCp>9_H;$-4lS#wA=Gtah}p+l;{8Yv%NJgk^dEp_8d>T5D^JwzMQ?N7cT}fP8?aHl zSDAMvVI0>rT@UjNm1IE5;oUs|VaqjnSMsPGm$HL6MO8E{NYgg!L1F6zkZCS*e|-=v zNH<)@u|o@5%8-iW4O*WO6q<2>FWh)2AcK#v5CFiieYP;O+ydA#fZ&sWf|6O#gPlFE zQXOpwrekSshP4DTsYe>H`s`XMi{~~dkVXZqa6$ywT!syj!Gu~C0fSw}N^Z6@svyN@ z5hy>l!uo!4wnSHiJ5PA)bTgy$@u1l(>ZtkS?O&OraRP#aW~G+%F_gJ{lT-ysKjTNcH_`$%kDfeu z{s&L>%8dT=fEpHQaHot>Uo>CFp}Qz)`@5b4pctY#_3ORw74FPg7i;G7Id{RG{YCab zrQ`_}4oi!6I|~#5nLHHW3cKP3fcw*BE*g&4_BVVUH5BKbOCpQM-X%TJtP_e@I>g^WAJx|98v=bio3}lVJBA{m2EG=~XUv`q~A&-LLW1Q#Drv zR9x;$AoRpcy-NJ!#mao5Tk3O42r%rE1wWwZ^c|@pS?h?7&U2pF&*K^#9TY^*--zuV zE(_>p6hdb?d&Sl;wTEu`WY=C0#8v?53s7USkPOQ91Pq)>ks0&^FF;|#0(DLg%$5S; z(82XR>ho#ajJ`DK5qSs*v@Aw96|4>Y*-Wgk4Ajl~pj#NMr|8zF=R5avKx?_@^kJn!k>K;* zJ}pL;9h0pe0X)iLm$#=S&iKEI$OO#<|F?eFr`?Hl!|^JwC|AjDAk+!a4bUNhPU^B| zFo#b>%6m=?>*`*;5KkddGuRB|7&X68zOpsK(y~VYcX|M5d`1`GBkT$L8CLQjQW5Ex5uMR9#tCn)mY9@1E-%IOMT@ z1WE~YzKw)pkjx`J*mnr$nNB;eM74_$F0+-gI1i$K+>G_T5 zv-i1fUr*x;v{bjXf@7wVH|Mh(@T8KRLZmfI2%vVb5(!c(1f&wl+N6B2?}e>Pm@T-I zjWy0Wp(`LHOm{N?`xn{B92eyw282@~o1g}Ru${UaC0_EBYx~(zxZh$Zqkl+**X0U&8l zsDUg9w*)-bl8yfd){8Ha{bNJb6l{SBYiUsL7ZspChpJ@*;Y>CO0iQH_tCL)-XC1+D zUf?)Kzy$<9i_r6{R7u4A*lez$4@=L|+V)-T8*_+ZVAf*g=u98%=rHa&>yyc+j~?sU z7j-qHwP0Y@h=Mr>fxtK0vh@RxeL?oVr?E~MEtFvD=NewUm`VR6ocb2pXEh@q`}c9^ z_fVS|V4GohYff)B zszou^r(F7NQSCs2*LY#8lnlZe(Tp|zaP=guL2Vum9`w18ilX}TEl1>jZ5`3YI32zR zJpCXwbfrE{Q?^GctU+uiAv%JjHCg@-&?lp*AQ1_^Ky}fiK+J&K<09{zvz7q}9;|$l z8)iKH6Q(`PdVn~jP3q}4qbO8f_G(H!eTSt{50rNI0y6yAwh~10K|-~!<8X1d)2rTn4D+be{3Eu zLU+jqqTQ%(;y-Mz4g}MbQ@=&M|F!bI>Vt||vdx0r?tv?kq{7y|4|APWHSSN0f)&6# zwt)yJ_XjLj33M2B3~Z_@2)NF8Any>%mJ6gV!eG*eWqpP<-qbJX7Eu#j*#?-ytAzRM zUKE8m>oYWciMMI9hwhilu?9g)p(nODhko$SD{TF^5vvIIPgU$IF+C}cS;tIAqvbP7 zJt^OUdve;{7LNdf*>Y#QlPgJu_JO@3(UaT){2OLZwh3^XEBi)lA1zG-%)!@uLwKqsu zVE`6SrZ2<6@*gim!XahWn8Cv@<0&_%M=lII@yAP>7;g84wsYREqBELKSoS9W0Xs%> z;>mNbH;>q1{4iq3i9qY+W)36ZP@4x<6P{|*ZkQeLvAkc6JkHO6UP z4HEX>Q$v9`Y8d3xY|nywj{;4;jz9OM-0@p+n*EeVk3u>s3_ue0kfiPW`Hixg6hxqo z^2}wR8C!*zEDK@d&3dc|9rbRMp+_8=8Rgj{L!_$!H`oevEC+gd5eWi1fifyhQEQ3p zG(^KgdgU>-WA{2#l9nOz0y$bQ4$w~urT|!MUE7rx+7vazWF!+nBKBHQvVk0y#uNim z8gMTX0O71m9bjKe?XlvN%lh@ajHQ6YBFt2wXRBZaWe*p8Nw&}pSvC^gTif^ewB?=w zB<~&4&zdTYQ(2hQP4bLJP!9v^YyMr=W3aaXQsqG}PD{0r%RvFj>XH=gE2R2K6%jDK zz}qKr#41OCxx(K5$EMVO{#7Yx+qx9G=;b9cV4LIE!wr~sUc~6%=pT{Rw~aMcjYB!@ z$I8y+e^ash_UYh3&%v96Vz2fz1EJiubndoY2C%lYZ_24os?rf>1V4B8T{Ii~HeQ?i zqW+=~4*zhn%aav4so}XiVE#`_Vj2ULS5W>v5Zn6cYo+lYAC8^$pjCJBDKD0%JI$@v z`yMRqU2gJ)p4%3Wj-#6$NFIaA`)rvRWSpKBp6swQ+Rbg>P(kM_<%r3gUA-br?p069 z*N@DOM+e(m5a4yOeNnQ*WBc6Ys@xmG3w%MEzpeaHY7BGZN+VLjiLia>7gMZ@_K{b} zH_h{?1Sm8mGajZ&ihwnhj?kv9iB#B?c3APg7I=XO8sVHV&(~iD))DhV~TqEKNihFug@6rnkqH%i3;XO;@a%A`O2t6utWIBZf~ITg!(DATunZRa2Od zttk*B9%MDHRM^^5Xb4P#aqsO_5jLO#qs;;~t#~DDCfs2mbEqZ`EESS%>loWpHwSOt zt0W&}c1i$?VByKLQymHNO#)b{!0Z$SDCSAo3kdI|1*NCcHDBUaF#EQrTkbq}dEegm zX8Cx$^hWm;kkI?zGJMa|%R~)ZdG)@=fPy3^ z%Kn5&+2$9J0rELiemJi#OG;`+5>7^5W8Vue! z-b)UdC8wnjJ6sZ-o>7*bUO{6`c;Roo^ZOUck?W3b%6am=y!jtVUrNl$Qw%%tVlhq# zSoVr1u#eaQo^)v}Kok&QM6y8y2XiZ3a$;c*VPKx zJ>*fzB1MAI=am(hgNH)qUN)zv<-oU*8XXpBK#MH`U$4$Xm|vcMAekb6&Gn%dz#tX^ z)^}JH+;a$tA1X^WD187bq8b%%!9f@5=YYy0k!N5kh{V2q8vxU!INSzKo(Ui%8SbA& z1Gb?2eXiT(1;YYcNabhD!jHZK{SZ=*dJgriL#p#Sv?WL|th)v=OEtN)w(A-Y5V`qw z2k>n4GmX4W?14wX1-6Rq4VC)dHoFzM2(?Z#wo`;)>dZy?q(><8HB8?oj6+p)-D!S` zHDC;i4x=CkqY8A2;cr(Yhn7|Rr@d(DQip}GuspwoBQ4~3C^9fVy z&=Nu8kwFLRObv5LOL6v^P`SqMx&%iyHMO9gptl)*dPtXQ=8!YpR}YptP*=L}=d87C z6?>b+WLdm0W>?M(xk_*fc~PgQ%rT4Il?nu_(YM4lyNZz80>{}bi3%^%}jUM%@Yj+6VC*170Yr_n3*QizEo_*&S(yq(UEEIFCZ$;)|~fX*6lp2 z_vfpy^RX3__|HEoE;HywF)_o;)a)bVihDkjym@x9V6;u$dj)uE&x1)ZN<`}~G>Bl- z%6Pl;vO(HjKb`BvwZB?rs2K~S+YYdf=9K-ecdaw-4E-3lfDc7yfRR|MmU_DSjbpTuJWj1bi z#Z*u#MmvL%Z5;q$0WkjxLNbwLLJRJuGsuMaL;RjP5>%#6iI z*^6?j8zC#{^>MJ~h+P>*mdC*`d1vxk-`RtfbGT*9m+^P!d)7?fpZ~e?>is1k&QHWf z8c&H*q}=^xQx&48X4BQuwkj@|=Ic`C|AK;K_S9O;=D2y8G{~5{XS5_saHGO6(v}(I zk2(M8iW7>ssxcax9}P|Hu9WS04QZ5F8rpV5oR+q&H%FB^50SGFHiP91?`r59)_Z*{ zP*#zuzn^6F$aenAnP5RihO5cP>iI~E8j`p5?$DxJdZocpcNWjoEy{>=M+iz^0mn(G z0I!By?rq)KsUBk9SswCaFuT#*KByV2h*4-*b}8j&s>zin>c2<*ljTN(#Z-`~#~DvPp7`h7p=b(raYJ?m5NAO)M{+O<3BhBZ_~ z*Z+QfM%{FvhGbp(kdOuIFgiM-4P$N&7P)XrAK9QVul>XQ47)VfB&bm`RIcmoorl-v zPsHk3uv?Q;?TOkEFMc{)wpHiyJ$Ra08Ubf;J5wDTpAvJY*sj*EGEjY?yOiajO81W4 zTzg@+)pvl(40j=?JLgf2!sP$y1JXwMQFyg>^buWiluMDJzAyi6=1_32OVgHbVJc

(SLSlG$*prxmUN&izX_~nB#nzB% zqd*q2Os{5wjt}+8w^9r#ku}gQ1q-s3k8m>lph}H+tUkj`fpOr-sIA>gITILWGX|ZA z0F^27K@>wZuqGd*QY>VhiOQm5c><6o3(7c`EPX~oDFT`kAQk{?=`j=tPHNDWH~gl0 zxA%n{fC`e%01_@M*=!G>G+G*~%Vc+7Mt#U&J1_M}MMWg<(%>1X8tTi8{d$6t7CQtR zfe2Mxj^@^uWGxm8XX?6+AqE;UMlM%vbbj8yT6XZl*UQxqFa}-vvid8BU@gm4Vf?FO z-E(}tCl6d+viZ3Bv6)fC=dwJZ=(Sd@!gH5>&DoUu;CtdqKpYEL{`JM6hn!*V@iY$t zwpgy}?_j#|z?|pd-QBl&scuxIj*P={&a_%RUdW;IH02nYeQ7AXNn_iJQobq0lYZUvMN~J|vvVl>V)~l{WO+qb zOgGX*dwXoY?BdEqG~W1Gc+zF}g4y+Z@u$CeNP~&Ll*+z2XwqIk^_3kY2(PTN3B&Rf zNkG*l5yQ!WK4ho4boa3!h?tjhJdIbY*|7Yq^`3!7$nWWf{Bl>IYn^`ZS?BRPqY(Xr zKbzba)K%NR)_0uU9s&B|2Q04j(K+lbtt9>GHGF!VN@q~pc27!!NK|Q`QUJz>;9#{h z1b~&Yk$>c0HieL2TBM$&$?JNaVIrCWNq}QeSeVXdu9cn?m>i#lYo(NCe4B&JJF?c@ z3thLCOSm+q0ln%tJw}rajFW8OL1runHi3u~9WwAM1As+~^+*Qhx&oI0;xC0l0Idc= zZUBIU++!t8nU%V^b~u@e+pe&WpSW@PsoR-vmur+l^*-Nfrrh`z9fR_@K3?2ye`BHe zUB6#W^ecvR>h-4-9nW2$jXmkJ1`E>Wm$6mDF6y$ng*TF>I>;ha0s|EC?3!6>W2XzF z7*)`9U+dzovqx)X%WGBZHOq!}T65m%`=y5MNOlh8x!oAAf3I(SE4JuvRsVWJ{_dqq zSYqqxGq)Sh7Jjk_x3E66Px1Ol-J#4AaUyYL)6gmNx6G2-^QB4Vp`PHF;Dj8Mw>m-q>j0jL)JHBffZ;^?7epc+^2v4 z`JAm1SA*Xle19eXX$RIH)psVJ>7fc#u{+{Idudg>}AN*%Ic+>e)hF9$&Ehaq?Gafo2;(>IS%! zf^a5%!DtV=^D!f0%orIv$v@hS15=$bRPh8wf9jhd41IF8SHG^;?a@m@wIeOam5*P| zya@`^R^NgDFZKI7=C@_8%Q|`JP8Kwpi5VH{XE`JPvqL_A4!$a!JmcaLIP$F)_wJ9~ zq%n5Vy5Jk1FWh{L@{Sq(X9u5SM!drzoZJX8Cfnj;ww)^o`nHUJ&zE}NQ15sU228VP z`uaLyROG#$ee_QSgFD$O6otPOdE7N$ycJ%Mm@(p9DH?d@*5ms4e z4EJ!1IO(cpEeq$!$B{eyx606=+I=7A{;YP|e4|bJIF%+rNUw z08b-Hs-F~o$Ec}!GOAqE=V+@+D+Fu@y3=)5>m?14?NcnChm+>SlH@P^OF%9(eYcs!M_TcB%ieq(KTX z>X%LQzqCzMd7##420bl72RunGp6WLuk?Anyiac#-ql``dW|*h)$#Iz{byF*X!)+;#rvS2qG zH_4oMs(*TqopAfuSXJEVU7yDp$va`MnWlHg_mth*dv$GZ%}$Gk(-x{><_FiRT5p*> zY~SH_dEYgrT=Py#xHbs@Q2Pmi!T}P1Aq7AK0@p@7aGWvmo{4MJ% zLBl!FFQ~RZTYl$>nZfqDC;23sEG4Io`lpw5=`DdzI~tyqkdF;j>Pe$VD$GwW%{;wz z?RhmN1*PoV*)&$?c-eSa7(|lVV@>hYZHfzh(c*XO#M`0U*I(UaJkC;fxnphD5i!wn z_}QHsZ@QRsLs$0|ELq>DK3A;F8Vj>~pr$~ExOVYpA7{uI92qV(tlS?d|IT9p7DP|= zg%|5xrtKKe1J7m5@9f2@Qj;}5NEtEnPjZ@nY8DysrE1yHLjtM3U#d}TdoigH3sb9P zH!CDZ4?noO_E}ixQhoj>%DTn!pSN6s(gy{h&c9g+RsQE@-mvl$iH#K#U+`XLjHZxr z4LoUj9yT4K#{FNy9&<$dV;uad8bJo(W}>1JD4+hu%C$*{OM0Va3r({9bGj@+Vd>-R zI8vJKhYZc)h+xNaKjC3n!(rZ`Qnpp*IRHQPvVstvbZwc zrXh&)i{s9%)o|bm!nWv$tiY6N)1og{5w>4+V__5f#MH0lU&N0I4u-3z3!QDJtY(Aa zVrXW0|{Gn~*4lg*($QOLV09H4fs ze1F4ya(KJ~rjT3Eswmy_iF&17@^*o+&8^@&wF~HUrp8B|lOeTYmrK*|*)v}qoTHTtd1%6kudDaXwQ{jfbxsttXSfVoJJ zYEt;tPD#z)_LDLKYw1#J-a4bS!RB9m@GL_2cd$}FQnT-P#vh9@jjpHKXO`wmPW}5i zr%x!US#~$zf>`a`vcq2=)4nlK(&*}G^SxPC z&ZDf~uz38yA!3=@uHDj&WQI*HM)QMX+5b3PladSTb2g|1I~zGpPK4#h>49#CebE?h zyValV`#j6Q2z%aFqO#)r6MGr9}RJ(Cb`Ihc!_F?CFlM+)q!>WR)^m%RBjk5&J zt83Xwz&ZU)qGZU?)1xRLQfu4Kh=aSA>?+rPf4GLY>|~}tUhwG)1?tRY?|D$W?H0pM zmdN^_cj7lyg^Unwt!XF!MAPaz+e#3IiU~s4$YOX|HLgK9GI1h*`l^opZ-;un!f&Ob z;bz;4MHj3BIJT|wn9Uy3@kVBk`~YEKN2gPLbr|asH1mC^&)tJ`=XDEQRD^R`EYugF z(Q;mdSL+{onZ6rRc(}I3;Kms_=!~`O{rZf-4t=-kdk2d_?U#R9zF>0)UKkS1>Qqxv zv@|oQCz0O_dgLtS===@Ba7BPS898n=7I5j$vQdwM}AsK&8;}$`^D%dZqp)4r9o(0k`oN6$Z&izwoNH<40^;DWRvHl)+?(%rTi% zD}nykP_p@g0PYr}k479*Y0imO-A91>F=Q(&-f}~=evX8;?Wx+Bv$k9s!yL0cM%-7r zobk$FNC=9rzB##c@SosTZ_d3>AQXp!t>Rd0jBfe zlJ06PlO6uA*=}a)#|Xz!Qm#R&nVt>9i>*tEhBc}2S*=+p!DGuEjPU%!B7e>ZY&P=h!kBY}AWH z`n`IWDELa~*5~Ldz+;jaFX7`xijpV> zdFjx2VNWPfW^kXI)6v+az=l?{MbxPW1_Gkk9Q@ZO3x6z*0uX#O(3d1T*?v8=@G2>k zmNLpY*PSdTvL184u=B@Xfd_1BNze4KPhC}ZkH(n)H6LK-pA4K`_JNzz1O69V1d{#G z(x++lIw3Q0S?#l14RGH@V4U{Yb9n^JLS^10-+*v_OKsQP4J?h!iK^SC!?Td=y#9_Y z&Zb}GV(^+17a{?d%$mc((jNwG52L>KyiosE9jbTp{B5hfe|PtvirT;%0`PpU^QAlI zYr+hCiAElZ0RewrT>QszaqYRtoIKd0ef;4z`u;1} zIaYE8#B2Y{i|q|(T8P2j@n@BM!-BP4fK8!A-;+Te@d{j%$i8USSvS+hKri3;*m^Z0 z!PArkUfx3(6vQw3_;tmf%q()8q+#cn=n_pPEzCJn6HPwj^5?O~-m{T!)8T(GBbZ~YW0s>DLM->;HD&4?y;rLY=48;SGbTIj6=|lm@8lXL> z?de~7Iw75=`zf6J4e^Var27Q4CJv?7BtyqAVgb%p;BmbXZ^*oGAR&y=bph0QM#RT{ z5nyK+xOa#hpdidWCg>&$Ct_mHGyF_>=xK4DL=0O$4eA%89szKl(l84wj3qz&@@~%! zIp%-IP-~RLH8Rv99xZ0!mzl_`0Q}1W$3P*YJ{*%m2j;Wzfeg|bKL%ue!jOf}*$t8f zpv7d=$n8im8NbHEp69Ct((w6Ys7X40gKxT zrmvzmxk0a=Y>O5Y?)JxT2$K~(fw^S-7J=tAEpeJ=_3;{NnQ?hpj8O8n%4OkW0BB_e zJfG!XGLKSTyj;Q$z|dgJlq+PlB>oWv{HVmKc*DPgpZkf0 zp>NyUNW)ZY4U-7+5P+NE!@irK-_W3snAkU)@QaNAOL2vWm48YUAA&6s{c!Rz8IZ^;`5VtD2 z+`{soWy3f4NFLPL3y8nqkG~7ZEard}9ZGuzxKwN4){!7g4?U6KG$#)2AmDA}_9idD zFa+6U;cPhi1@Ibo|qXL>JkGd z6yw*R$R^vWbs6qnUm%JM&0rx0Xn4d^=N20Ne@tj37duVBOYMa&3$YJV5Fg1nYZ05t z!q<^5m531)tkap!EM&N9zNolw5;#f5myn@ME>_HfexCq7nZiu-P-#5Wr9nU!4L=P) z`vTz00-Pc5fDR4&k*oQNP|lO36X((7w3AW&pbREIjojXBVizvNw~%ood}J{n`Mn0v zpNMu9KzjlB<{m3hIrgpy7DB^rGLw2!eYe7YbXse?7}4yH%V#1pxNUB1@T8Dy z&V&lPlw;FxEDJGpVniK5H)tAjo`#zgqnTv3%|$ z3BcbKKwKF}5HQkRJ4h4Nj+PQSgcSCzr=-y5YoYpz@Ler5ElNB1z{-B ztza*d7Y}Mi;xmM|5&%}$Lz_CtP+u;3Q+UHojR!%w7|H0~={ZAhNXDeK2k_zlnd4MX zB659U=CsyclKg{~)(jmqPLdfVA!Ya_@&f?@qM|DUOqi>n&fmcF6%X6SU}f-7ozDOn zJp7R}O4u{-Jq>`~XE!ys<4Z(I^dN|Xg|q#MWA0(QPy+teSrA=>{>Acjt>t}dLFEgP zm;8VxB^ehI6fj~mwt3H-0B1%}A@W*#yjxv`L>(G#LC&!vx#Sr1FLAH2FK|wL#kD3W<2BJH6RK)}*r1*5 znh(mBV^v7Px0oG;=qz}&9wE7)pqNRQu)xct?8ZLa8WU?SiYd-%*AZN=6XFAE0sXHK zC7O_mpCQVOCI#)oVc-3TLd+__%FIENF04j%xyTBUU3>>KL1y6t_$CdzgDe}$!^6#G zCm9$sf-X^r2q)0*U&Fp7jihhjud?v@d^=kjKF@(VTRCX?1dz4_1M*@zuVZtcf_lki z&S{9L@+fo4l`a~n=5|0Mv+3nBjLGYbimxS7JSvu)Tnm6{Z=k3VtUDLjPrm-C8+Vgt zcZr4L@zl-(a2%nNf(Q~Csj4wl!J~M@PKD11(N{a|RJUjrcRDxfu*k;XRmt^jZ7h7} zc3ovbPTa>+KEbVaP1t2XUGxo%ZGz#R`jj>p`m;WA9>>~@)rYvylqvHhusvwM$S%l(C}H|a_!;85;1y}0-kvW7yZTu|gN1MNFAEF8_`EJWJPI%bh-Pa5YJ2gQ5H8XUfO;Ojiv$5&OCH%1 zI{JjzOU8yP`K6&Dref5LfcTJwc`G@L`XXpqpP0GDQ@h2&QCO)r0~gk1z@mg`tFv}m z+;QMF_sIo1hJ^SfL`XxcT}kp`)7b5x>Z@taLY+XjSBSqPMDttX(Tk5hc;K%BviNqs zQ0n}z&)ZLxI!P@2>12EdFOgP@FJWCed;?V->eO;^0zZu$|Ko2@`HW(MmTqP4xqfd` z_*&O)$$dz3lOU2MAvaiv^hsd91#~OTo}63%19+`Mh^!MLuQEs)WYh*%@z9IeXi^}G zfK|EpUX}#vTEF@0E(cnMtuxdK;K&H-d9I-X86^@3VIJ|l0GyVXvp#{&egph4 zLj3sFX=aN`6LrYRLUxXZ75sa5c%^~e3-Dsr|FyAd$}{nq&XI_3{^By$ZNPs?n19%Q zC>1Q`epnM4?maNOM#HTNaC-w1ivxkVxva8{k6s64BSa&ACoq3$ILWV@-xpV0-+qv} z)?~1U@RW=~6q_c@XDZZlT zjaBnK@?8zHDSlee|3+zBS>`-?nHy8$c15E9LT(x}zwvv3bvCZwN<+Zkeir?58{o9; zLF>uSsgihwG?#czUUYh1G>?yS*(0mK#P(C-`}x?a^?$ecI0|Ez4j=bWNUS0T&`ZW; z^u4lC1Z8>olqbp5RK3kWJwifMVEQFO&}@Y@FgQJIjnu*LsOh!x-{jj`u0AI|+P~!^ z>v@$qYn4wt;K_3Wzu!?Rx6g7s-GQBqA+}w4CdTI;NLViHIF|mr$Qh<~_fMVP+IuVS zjivY#OZF^RRLuJ1A&b1L2k2&XOUHV7;E{aE<_nuXR#?US_jY4TY=iFN%26v@Rj6LRyifn{wo3K@B)0QE!$*h@XQITjcNO?Y|nK|5F45-9y#A=?0#M5ysl4N~vO*lH9!ZLCW8-tj`-A_@G~G@-vz+BqS$0^_^@El;HSKgq-_?VnXVYN1bfoH_Kkgsja@FSL19BG4G*Z=8G0#zjrEPk51c<*i&F#X;9^(s9;Lvwke{3dGd zkZ#suwS)Nhyisvr$9SNuf2q#4Z0QD3VWN0}k%}P>rZu4`yX*;?za!@{61FZE8k(&~ zYf$woObM^lb4{_SzC_5pchI;6<%E!vyaubb@aQbovQyI+t_@U>0fSp)QOj$ zY8d~z`;n|@!JFkg3}k=Cq&c4tQE`dz6ur^EIg$I3|NPeVPSdHHGPIg;*|p2hZ;cJ; zL!+D15p-F>tkluA=4%Ry0BD`c8RMR0gykoN)YOOam-8$pB8J>|l(ppOR5`?NZXWXL}?`KTWy7O8>m-mkBV+B+jcERi?qmV3x zg6&8D)IoGw2m}23imswM#hqDWgORq7E-wnumiLAV3C_9bHLp4+pDqcK-oPK#W9h#_ zF%s{@Qy}9EmH*KwI!o=`hh!El$c9%V=xF@(aY^H33R)=uq;-_%YHuaw#gm={9u;Z$ zN)~)+@+7v@p9Q0$4~1C_)Bfen-d#R5PZbi2nouI1%G1pPKU-NFPrE*&>MsDZo|C0| zczA2S%uZ=e-@kfX?LhP_%DXg}I3RRTvMtSxv9$$y^aOYpN<4G}{-%M77t+-XW3O-{ z2BJ3Q?dApQdp`cf>_;)4YIyZ>>mi}2jc&ULOuSSnq!6JqldZoFg!kqoM^}{fS|+V3 zmL@^iM5Z#XV<4S|{i?dBh6N&wImf34fjK2QKR9QSHI5u>h>oS<_` zGz)_|@)hO=UmkXzLto&plTnlu!x;f9GvQY4)rhB`|JCGHCuwfeee>*4qXOMqbWBEe zCHtT}!y|#K-sqc(@CdqhLq<4~_Nv|6m~p!FQs9R~UyJA0`SmL6{oZY-8%`cxN%Vd2 z)!_oBH(Wy^^}ipGC5>DPT+LZvtU}8HDC^p9K|pdPS@9y<{H2dq;Mu!F5yed|^I|v8HR${iTwHcdyL??XJP^VWnOE)_(k+GmCPzETw3M@!_GT zKk784)>?t|*j!#p%9?cy+XDghyEA{Y~KS_5Z&<>H@SjWySS%TOTv zRpm9Xs(1h4Ce!~@be?fZ{e2h)L_kDEMFl4~apc}30deoW3->4u_s9xx;mW;A&7EnQ zqfDEi<|s8SOD+3{wyeywtZef5yg%o~?|eSLGrs4(udBL`S1kih9XB6S0>L>n^l~%N zNo%^E5KOp94UZQZyv8E z)~3_wU~mH0W9%7h}v;|9hkU zOg~7*oU$>h!6=S<4TK%TLyT@9%O@ecgMT+B)!n+~O&+%+A~Gk65Ok=O_E`wSI72t* zI@E~L(%M~TtEahGkQkl=Q>$l8fr&hZ=rGyiT@G?lHxE^VJK_q@=c{3LdE>yQEi)8b zl}X(+XTPE()X@(m-|RkaS}TtgflQ^Mkr^B9gg7`2N& zLO!Y7tgQ|l0SeZlMaZEyTMKn*@J~o6x|mZKO)Yx_uPed|CL&MuyD800L!|v^JXYy{ zjUJ<@h60vVHXIuN91SY%l~{Lrzcm0^ZlJNZTNAdZh#B zo>jw2=&Kh;hIfSW9LfE^@GWV|kLj|M?9 zJQ!P|BuH@Aewi&y&@=Ai3FJCNF{cdbSOyd}EwOnd0HGEK!iKLSX@NR0Kh8N-LiW-A zVO^vuoXUG@Bv+S}{kO0Vpv#M+Fajx1PdChAY_&U@*VB(^P8A?>c#NaLsa+5vE8D`D z9uwLb7!CHH&NFc51MT!ivKUg)ISC_q0U>~8oa`Pz5p_~T5)DnalY;llh8goH5ONam zQ0e57z|H(Pe|a}z&OssYPj^knuY7eZi0e&(t~yp(Ai>0Ft1zbct9_^@1?tz;B}u~i zE#$dl8LHE{iRfIk2q2ExDfC5=Z?HUaCEI4dW}OdUADw@c#n9ExJF%O)#)fC@D% zk4EQZjxY$L;B@UCzF)klFsY&s(9k-Nrd{6Mo!qKbpb^vRlLxP^cFtAR;*60zXIid% zdX761Oh@NO))`riG+ozij#@LKpSOvHbVP3*^tAq^>mxbG79e39 zknd*x)eL^k{>Fo2;`-IN=Qi8U?lcoZr3LKHM?;DLD3q;43 zqGM!E)q%w5Jn7ot$$h9i7Ili1pRft8qX0Z8&_P!Iz{*VEh+0q&z@L(L!|m$LgVnCi zWDq8=Hv`ibS~rj3&b}9s#&&0NKoBH2eMg_}Mx!T|Jn6~L*v$6|r&AOv9)&fJkAT9N z(3$Bxi~Z)8b@_p1%@xgZo^{Z3Be{ug1>w_qX$&5ZrNY>L&1=T=^c~)6W7K39kW_%! zT~WC{sQLm=KU2p`UCBFW&rvxLsy5Wnjphr;_sYzTp~xS_=VhwqrZgKb{>isYuYe#y zN44j^hADAHpFuLg{On)lM*Gl5PPW4>gQrAccj}-Ui2~W^bD90-(pcEF8?se^^RI&{OH1FZ1w;yBS!7rZ*y{Pjh6NP$1qNdB3t|r3z)q{qe++LT8pjQFLB>U3CtFZ!3Cy3Y~8= zG6SdO9-Qb$VU&&J#h3z*YP$y>n^a#Yutf^C+{&}S7Wdr9z4qGCX)Hey$uMK(b5GYj zY{wr8qvFMB&{If4<>(JGhK!vea%i( zoWdZRUBU$moBx8DwY3raQ7dk=__FGh-F8(bBN3-w6j0C8+h`kFM z=wW$jB=>uv>Q)TA$GY7jb zLI7zBypDcS!4hoT7w`{im}$6uhQc_@gsosbm`py`FH3`eI=^i$k1v!9SewTb+mJ1S zmz;y2STFrG8?kYalpHwA+?hSC^hoSv_4yzlv4Xnn-WS~2YVVR?P$O-Kv0Y`CCw@;7 z6J6qeK2|2xO=h96`gq-?aSXk7rR0J+Ea>AC<71?`d*~{M!w)Tl+JgJ7x9~YJLLqP3 z(@TWel2Z%J;G4`qCQ~)*rdG($W?#&eL=)dn3Eq*HTK+nV>bDKfp}b5p3+h0AsyAk4 ztDA#Uv`xpZ4?C2ObkMvqUx30T^6>DC)!NLMB?rEc{F0TF)fehBo%@5Z4l~YqF{a8_ z>7&1|8uzH4dbOPW*Z$P+k~)Lh4R?k8GWokCqeDYuSDMJT@%mZ&ezV1OtjF08J;9%R=SDAcL9+$3hh!_?d!6p}0iC{ms1+{B zT6f40(I2)gA6$#QRv~=@rUw}~7N__s1TbkIAQ$g#tth&VLy zRqDL>YFgDmd!zpEO=s--rQXXrH(O^Lh0c`{`||DkmkGDDe0$pyezt`* zMG%4|!X9|BKP>+ss+OolOth7*H5SB!ms`7*m=sAQ? ze8B5i-tvFEUNOl%BigvS7x&|BEjMmJVzD;*k*0-3;djESfluUYMxN66-BKcL zt9^lQ<&TO*n+E6{>nyFT3BPzcw(D#xH~JO!``a^fLd2BK-_ai^k?(EPvID8i$o)}9 zZ*cZ)bU>c^C*ZPU?}t8FkLX|@^}fh$k%;}d5BvM?$Zuc%PJhcczaG{U;35#3eA~ZM zHZ0+Aq!nL3`3#L3js+x_oaqvaFs z*ylfuJpJgS#J19{ypQ{^_Z@wok^(=jG(C^VWpdYZl|!(a@6A58IOCtm*Ur?Gu6^s7 zeR}Nezt@hp-z3-=x1JW6zT=)#w1GPK>Wf!yuK>WKmF5q zc2hyc@Y|YCirF=>zxq!6zVmtMg(auAw8}8}Sk~L^$q%sdJS~l%i294< zcubaBekyI*E?xOv>eQ!r@-)&qLZ{eddB3mpvkO&NEqLDVwg>+0R=|AeMAy}CPv zfO$u6obK6OeA(N#1J~L`$Sv``E0tC$8;LW1|9t1c+xU3zW54&QsH2I>ClXc8e7TW% zY6@7i{+Pu+npZX)&9pw`=tB6Q0cR%4D=gUtDHTt$!U`qVYe~I`D zNv2nl%_?6H9_`u$b7(?cO^<9;!p`dM4@y#UYCq}S zSMuFtr-#YTcTZTp+~bK!{=2#NB_`I@^~()V3i#nyedTXbS}DF~Qv6C&{4acSmqH!sqvpvPjFMILTL$dX^C2CNtS8Ju4ySp(^5~QrJYGjFHOt1kd}ER?d09GtcPi* zUZ$P?oOXtrMiWY>%cV24(z7kob6nGNkEZ9HNY6i$&SjRS7hFg$ypmpYH@)~_ddbW5 zv!ByTx#?v>8Rc>r6KBHW;A`y zxWLV57Rqdq%WT!kY_rU4cg^fLn%Q|G^WvGzuF}j)7c#rAWcJ+6y!=%L3`wcJ<#k&GDrt%nJc^-n9XF$BE`xiM!p>SS3 zSUYDS1On%g`d7KKuukTM9+GU>Ou*U{$sdZu&rn121re@-;l}rzF-D%|5Y~RF5<5p& zHQadmQAjRa&i3%@{oS_7Y9&ieivW>U_cB{mkH+b;!t;V~s>?UOB?W%n`0bI9gq^o@ zb@3~8*1xZi+|Oh0`ZS=geOzLs*TdHmY}I{lCMkV+J<9E8|JR0AP)y)%vXY~pz9efL=o3V5?5d3v>S@@3`(%J$?bpy1o0fqu*4(W7Cl{HDZFCAMfJ%j zrtG(5UfJ{@Qkabm^W9T|SrGAPh^ZTH)tSKNzNwC~R_-LA%m^qy&qJzDf)X4T4_vT8 zRTmX`;S-B?F_c*43l8QTKOYK4HQ->faV<}XmAI5oE5WXaauv8Y%?a~Qp;0s45OchEb2N{9p+08N(?^X zm+l4Rk-?7LJ8t#Fr#53Pg#zZkC1a3-o%nFjxQYOnE@W5MCkJ>^%pb9-yw*iXSt`P2 zu_NS}smhUa_7m1lB|#yN%W)suE$ty|c-$s_`%QKfe^aj-f;9|b|XGJ1t*4gQ>L_S%IcgN37h`JtM zIj)#-+bu`T0MPmfQmubasUqUI8{f(4_a`6%%?V6w(IZ^)8yRZCpJ?0Y@99Q@#J`roM#M?+ zUMg3WiSd5NgPTOU291RLxOf+OD3v2X{HK`{rP(QQ`}6m9!h%v`T-NOlwf?S2*@R4y zUzjbTS^qT$FQP-^T#;(O5+4`D^st*!KQW2<-B1q78)BGSZ_CBA#(k<<0SZA3zNA6> zOQ+u)uB5oeA-kK22N?(dMH_kb5h66p=P>tPU%cddfIzXK&IC%y88valovd7D0Rsi^ zJP@`)^)sxh*vDLVFT`|ilvQ2TGWbvZrFF$L4UWeug^!@=`-k7hjAMnsN0xfiK=vQa44 zR-#>4@mM@cWVtKaG<&2)Sf^iFG~I5jq2sk&lDfm9uKh$L?obZc!;PzWJ^MUwuem!b zyCZ(E4E(epno71)=9>7*N+a((DNR2Sd)s-pk~5_kb>B%ukFQq`SE`_f17K|EH}i_$ ziiEG6w>CZay=zy(D|zg-(u8M)AT`g#p_kL6swVC1N*IiOzzho^Ptq zr%8Sc8*NADYxj@#O0x4BdI|A=y1RD1Fxh>?oOE*>Th~m*`X*Hq2gg=fIV+XhHsEY+ zzB&SY`}+zcHGvp)2HNd)rbw>iRx|K(6~E(Gn4^C{ zN3^9oOk!S)+x;S|AWdC51iCGH-A4(GYZ4AoJ>@5xS0$RJaXe(tYe{XyZUJjtHX*j> zdoiQ%LmHXCQtfZMPFn1ihS4BQR6Q}T$GOziXuU-F*c%O4z+zFS#i7`1U1G`S9t)V+ zJ$Ksz4U6EPG^5ZFsANT$@t$HDv+LQ32X4r`44aS@cS0EgVih16PZ&3S%P%%x7tV{Gknt-6 zzURWagf&>~R)Tc5b-@T2b)fPL9~XV?_DRG%X$>w3Tom1+Uw3Y&33t6c@+no7$K6g_ z{=RO-J3dl`#Az~uAWE{YB8oUgvjoBHJjcBl6(VrPaCCv`sL$lDJqC~wOfXl_8@nQd z{L+fYF-4~@9u(2XPG}|Y-Av`c8Fh>$Cg+e%eGVU_Q&MX4RtR}BWA>+#6->_RnoiG9 ziSk&aLaS%+Ojg~j;%PhAiR_^V!jbAF##WpF-jE-QoTZk`twl-Bivvs;vUDL~f{A20 zx{>ZaXpoZKdEN|(@l?M^2FoLW(95@1>VsfA{Rc5m<}BE!grIkfNhXsK z;sM(HlbAfNy2$B$!OA%=qstSeO+GC!kS&~YXzJ?=`vpmsRo)12wREA{)2#XCrQRPb z!?V4D>jd>>MnoB^)HqLk;-(pJ*r}BKFm;LyRlonl@w$wdYB3NeBl<++AdeN_>N{F_; z7vIWJk4B1z(hddxt%H@~$7SbPV9cRRp>z(B@AwlLu;A_@QW%VF*(?4%Ld5Rd+G&)6 zeid6gv1R+&Kwb7$1MLXxKUDYw!RcwD*Q^I-$w8krb1)sQVu|VyLtsV4!WRl*IiRgV zY6Q#aV4wfxIhdXx%76+~_VnxG)Alu1YixxPzq3_b!uNtI&>Bz*U3XqX(1>1t|C4Ww zK3@dV;YTcuN5ZO(iQQtOwg8Y1g(M_UHpdT2cK$ELiL>I=)?bh~z*3^nC-)}K z>ySvunveGRHxN&(`{}r8@&SYjTQ%prK&!b2;HLmr|5Pg&g2c;k!d?9J(u{w9tuX!1 z2|@JJ2|&RKX$1}-v`hL;KR}?LXeuCAQpFbRCqVBPlu#-r!U4%4z!d-qRm_%05P84= zaXZGb7a`-h7amWyrs>>wt^&w_xeT!?)_Suv*_{wjKjSvwEF#!-$P;fE;Og;RXU?MN$Ai5kQCVn~!Ck@@$*{XwH`utrV*d-d_y{ z3LgL*w#h39w#;DQ48eQbxJ2*YxFjFn^Pj<8TxdkQIM* z+rb`_Bd9;-9T2Hq(@NH7#Rqs=w?84gMxP))0IxSHLQ3oX62%;WGA_cvSKB#CZdduW|$t zsUXrcf%90lMAs(&;SmU8zvQ*c2H1pP-@maAX~Yr@>2Mtn30@~)fozT&wX)Y3EbTW| zZ4Jgyy^mJW&HrYvR6$(zL)I^YL!_0jI7vxr2w73-dc9zsRIwL7J74t!6ef;vM%c3L z0fkS4mQkUQOK$(tKRbhzpB~Pe%jK4=tGOK2|Pg19AtQ32I%)* zTGJ;YJcbV`NRkGL`*r#($b0ZkJD{it>7L~Mf7Xn<^y9>c8 zNH!6{#_u7$qf7E&;`DSCyBczgK6A1PpkVi*qmQKoSdUEPR#jmf`y(_HHY*+{w}P<1 zX*CISgJg*61d&54`3a8eJqQ7xeFGLmdtD5+o)9PwYRY6{wSl6qp_2>AJYL7p9T$u&l-MWVxza|@S;n~D)kx8 zA$}q&+HHc7G?j&E6`2pH?nj2a^Nr+(f(HoF{R;8af~hyNZ?NR-&0z8br%aW_=QgHS zt~lRcB31tSj#m!oc-1eUD@|Xdt6Y>7WA}{t_8EA{uFahPd+$>+O>S54XawQxfqT`B)(h`px^oGIicH?5_DK$LKwe1a-}u#0AoenV zka6DJT|n@bpw;ERwHJiz6*H2mfWd(1>TCT(?Q;Y-IVoi_kCQW}pM^VMFO~sZ^?P?? z?d&&$JjiZmJpepeiqh1MEPY`)))u+Tu5o00$~$KTYjYN(_4#-ZZ;xBhNZ;{;SZ!Mv zCzU4uRhSGTNr#6u?p>i8=@i_%VxF`R_6wbs6hT?MJ5uTF>zx+bw@BXchZ%9 zq}2t!DyvT#xAmMM+Y@Y7JM8}?pDlpAnF^OXy?3N>;nSJfxg{s$_YZTr7ue?fpOlU( zdLmH*L1V+FWI!GLnu9pLlEA}MYhgCxs#-t*23Gs^5mhyn%Zn%f1}~KHsn`iTmtiP| zU-&x7R=O(i`2CS-8JAB6fKdY&cJnG$nq*JZyk`eme|u>z{F*y=98<^pgjI6*tuysN zg17^0eFlRJl}%E*KED?npLzGm4RF)F0%!qRT*1ntQqb1S@`943*Ah~U3D5uZek8Z{ zQ`xy{5*_l${~*V>0ntHO+*Mxda33R>^KiG~zS!<))DJg=Z}7Y)OG+Hfgt@VPf2;Z^ zzjn4_#{AeUasudKL>lcVHd6%`Oyau|!d-L0($b*q=1VWaKd$_FdMPIMV0TsfKeAp~ zvc!_On^Ono_vMc2{GHufD{@l5W_tFwEQ2n@I|`_v9U}?=?_vdP5bLjHz@eMIuwc)c zCb?%g#yw8$^&K*LTU?4BNA(1V`yJ!hlU(cAxNs+h3Hs4fJ@t}E`87c7k2eE(GFT1J zI;VDEr;zlzWiDe9WzH}?);1x}xGwp&-;VrLRkc=R_0#?5Ck*U}x~z?Kqw;QjUbhH^ z~}K3vTl& zvw7o&1|L#1=gH3L8_tV-eAtl`CL+b&AD5>~!zD<4 zvQ*F$=`ne?5-bN$W@LOL`5hH(>-J9tSoV$)0q_tC%24boF1hku@5$2 zG0c~Cq>DGYuiHW!2+}O#trx^S_S@>+@+cGaX55$zn3P>iLV;m#zhAdPx&Zt=dT-Fz z3})a}_ZvRmYaBseN}u%Em9UwhE0Ukxe^xhn%K<|MiaP;Ck?do_;_~||N9LNbb|4SP ztBiSW$sFvn@1$Ll3@F!hl&tT;ef57SDmbkxj=(Hf?43BwN2W|Usdpy=Y%eu0T5W1>1i(mzgv7J|7e*6>Ej<|YJzm39$gdB_p!sP2yf ztG&-x-c9!)oB6u;+EDgnr#0_qVuI~yja3Z~!33UghID>|i(^mxiWdgU_%U0j2p|_j zJT-gW+&bO!&dB_8-&cXn-9;dz@NWGK!6qrLqz9!RB;15ou(wG-zjK4|3g9`>UU?*^ zdc7!@7{%$5c2#xtNaRx2>uDTHo`hbNwR7L%;Zkn$?LaH*r|QJ zAVtK$X1ClJb{OLlS%U+WKnN604NJW0I28Zlrr@XV-+oI_DeP=M3BxROLg>9Bfi&tG zeNxs?CskW#LUEFhEu)K{$cFMuX!vzf$J#hA%WTLf{KW3&tS{V1cA&F9L`b$N zFI3)K2mf*3{A(9@1(A+)k6vh@pLZlnl24$lLf7B|>iy~D$n;r3xP$(f=ij4kHfG(P z)W@(JRyAVknU>i`omk>48HR99c88UqD$t-tG1gUi;8r3`l^_|ymM>C^-Mn*$=V8w4H!`6pD|RDH0~^r|m~BI`HiV`B5f2qQ_IKIQ zttL$$-`Mwg}0c_vE4x|1A4MqjtmiX3HSG_ytD z492KH(63)zbqEX#G3Q+)ko&5Ug(33uvvhf#^fN)rJM+oe zmAIN#T5j!RpraC~wUXTY0R?y^0x%Q#T=84uxbqeLOP;B`_Q~xOn8uH+6nV^4 z@pJFH;A)>JrDn#Nm#*1NMmIm#l{7AMdWs_ClyU)8ag<`ouniJ2yXi9OlFZa`#nGM2 z>X=>u+cNbHLE-w=D?bR%CoVfk;SWacYsRnfxyp^kojBP6)k1`1Lcyy$4P#+e^QL1P z#MQi$YHGamBd@_rohj&)S#M=iP@@+<^# z{?Ardo@un=%{ny^V22c}%o!*ltC~GDf4^jAPR=a-1TzLR`(WfFYNUYmfoWbkS1o?y zs<6iK7kz%G-jBCA_VllKMzK{15>ZjUi6crx4ky|SGLm{clx>#SIIt3Nq4E2(xVta< z1YBtnPtL52v<^)%r51JgulSWPUdU!vKH=irq&(+R^z4%IAIgOJS;8s{mxizXpmYtf zBS_bU+r9q1OZL@c5-XZ|+xk528y>e~I``Rqa377+Er2lbWu;% z1CLv8k+~DNkl&`@i3*06d{?+J;nBnT!^=pcOJrQm08 z0*5G1!;k+GXY=kM*F|z3NpYn$-5CYUiwe~nW0{*ZeOjSFq09DdLArz-+x|m2AHv3I zJ07aiL7}i5sfVZ|!+i3y5oQjd%Yi6BzV5mZV)>_S24_c6^{Ok7=efktR@Ghk4Ao$b zu6adA0-*CfP4xURfMD`iTW<0_Qe$#^kEt7^eJ;_MKPKm_DVFaP8{i1Wm)WXyZ;`HaIRVABD7|3&O3P#N&E#0Z&!(#$7 zGX%a+TI2kt3QZlEqlit`zoicrEPjQNNZzc{Oo`+ro5M-W=1Vt@PbrCaPeYOG@>nJJ z!(|72-?CPp2Q5*&?ruv3|Ah0EO~M@x{)zopwP(6Zf9x|N`YGBzFyQgOT%hkt8yQ`$ z*chypc6QxeVS8K@FGF%yG~*G-bAym!1(kR1F>H)J7;63rgGbqy8ik4E+HBGJTkAo> z{Z{nL;ntSR5il!oLLR;fnA96EgZyWVl4pfj3RuB%G``tsV8=3!IJlzxS|!c<2^W~% zoAovy%DGxPGzeqGJ|**&{88N6Ipj9hvU95^Q zvfl=f-gg5!|Li#b7Q-j2IG`*Y3=-1uBVzvJsXk@Bj!X|@WBw)ZxgGLGc6|UmJX_Hs z(hs;*LdzC&?B{`n0mrS-B;Q~>6Mql~^<%DD{%IR;vKv#pHrN!zc~tfTxi05UK-C;k zgdGGY;K<|yiS`oH%E<%{ISx2Cm#1Jpu^v{!0hM079ZR$ixRW#P}SY{9N=FH!0XW zb3*CPy_JGdM`UI@5sK{8A{|wMoc~*(aMTN;e$fO@{&}9}PpO)A2Mh7Ex<2`?1TvCP zf`kLwd+?lf1U5rRubBU|(@&ViB2&$&EXVJqUhY6)N3;aS>*`6zhd&YPdS+O4EMQUC z;T3B%y{->ru%-^VWrNf(W}~Q}1%7@8EWG3+uhJJ;jVcQ4bAr_nNJV6NWE~Ot0iXD! z@;>CULB4f{wlFdkBJUJ}M^w6yhIAW3?FG-gB9|OhnPkw0DDVp*mmDQ^s-)VB?))C? z%Lt`qt0K#R^37LmrcB;={o~2MP&v`uE5h69EHQU-M^pG}*HbHU3yte5n+izJ$#Zm~ zUycAwOu6RdNB2<{6DBCO%gW^dRJU;D981ymumwWB-6Q^4*%{0Pugg^ffNA>8+y;*w z5n8&?%%S`31yo1_{4#mps zxyh|5`|#WGCq>4pFg)hw8xT zP4q~Y@adiamz+hlu{ghjZn5p9ND@%di6A0CKR554*m}xF5N|W()lL0(Snv!J8_XCN zXn(E2U_y}eJ4h!igDmn7$nOIX>Q)NXr9~~Z^+$7(Hk#p&U*#Syc;#`hWYCiM#z;Jf zpCoYj)+^=A%clz4*N(u1WpR90kaJh_gzR{o{8Wm^sLGKYh~}+aYCo`s5w*eO|JNxD zXz`H*a1?RnsRPqZCwN|9U*TzhCAeCH0?$)*61Pta{w7yRnx{~5J(okEhhM6~vy{74 zAeR_XlM4G2B#Si))Z;!3NksR~3INva9~~D~Y&mrwV0Iv&VDy&c>{c>nlcGN)u_()j zVJt!Mqfiwb!W2kYm1^rljzHyg%)Z8Qsa!uK_{5eGK~1W~GfOu1P|D(z9A zBnP4iL@=%h#7tx;I$1y}WWwAGj%=dl#tj!-<;rzTx_Rb6yMn14y{la*p$uMRh%-Ww zX*>o{VK#|G1mgknY1RnfJq#ZK$^`)Vd_qtCBZ?q^T=EqjRx@f~QIsTnLXxv|@PyO= z3g2hBqOg}t31fGxwF2JBo$B~u`dH7>^XjcG1IQ~KX$AXVQmI7Mkvuj z=NR5j()=CzBD_F#uT@K#VH^%B`66FTQa20kFsS?FT`}P;s3>~~bQe9w8@BOPITu$! zp3qo|mdy_d&};aIgA})Va0|n1kJ#GGwbB1cg*wB7~WemxFGB!;*7YB=7@f))GOtQ|XO82o* z##icDSx4=U*&PDKRJ9Y_l5v{)anei!M+c`6s7Vy4#FXLuY-wK>S`*!xPAVESweu>Jo;GC*=VVx%1z&#ul!i^Qf7gP6}j`<8QMhQ?l;hj6zWIdiL$LI%(!+Fl%)RmYCSMHN^ZcP4F}P;} zN>2BCVknT>S)6m^X6L@~bkIkJ)6cKIze$lHN{%L}WH&3ViffUdNFLsNf*Y&}kEc3g zJJD)1e%Y4Rx|A;rLyyI;p7(5>BE3D*s`bVVuHN!|R{Ytq=YI64`?KW8!{5V~{JcN> zia8#k;QO+h#07>qm41lg%4?pp33a$11g)`|^EBdCYrANKhLQB}qCR5ZOn9mSg=0DZiCt6jo)^pNsFvFO>h7eT|I+A8 zt;ER_HK+LMf-nkal15Y!bb-RITza)}ZOWm@y!+2kPpR6nXC}9`}33$JJ;^AXQm66hN zV`j`sR@9HvmS6K%Q-1vNNuR1qM`4UGRuN6*Li2>DYWrCRKDfQlCWAkUrk?tw`jp^) zY9sxB_!$p>_~H1ZF7glrO8fJf>Zh=jv~x8!De&x**wY zuh$f1)U6|VsYkM>1XK1Pqig6{PncwN+`IOl#`UggmrKv3=3no*Ynt`&S;}%iuJt#I zi_db+9-QEw`ql0Dp!a3Kl^?&lPN-aXe~$L~MryHXA~aBEr2dk9ZLiex18qpn`md{0 z)AGfGzRu0!uje|&q?iUbs|!?$9qwoRO)FD++Sh{_lHkdDXhatq9x^x6`E#SsNFjCj z+@;+5h8LNg|9Pekjr2qWp2CC`37ltKS{*2r(xKLj2uTebcn2H{!yD^6@2?C;nKbb5 zjPRXotHqCFVF9e2-EJ~y>hpt+g=RFUA%~PhL z(wypAZ@kRBFk}*9M&nEMI(1uo;#+20X#RtHemYc&=uOE>8Tzp%p$?tko9fr|ul&AQ zBBdjHcBnsnp4-6N513vLWb;{!31#yoI?NB%>4pv6S2E?r$ny$h1LNc2B#WhS^O|T% zHV;Oj^lPfed3+y7&&Q3Qz*{oUFJ#z*n0F!n8* zeKUsppTTssMN8Yz-i;o*1ZnSbkk7yM&Ya2zF_gJ_0&MsjSo!Z;?BxXaup7otZ?uE` zbOy5%;mSDk4qpG$xfG$XwW4)i!GGU}KbvJqY;*#61f`u%Bp57vK958bVA~62ajUvB zw-yEf3mz-gz2>q09C&fo@6e*>B|)xIp^xrZobemFXE0R7FB3BSZ8)Nq6_*5Ep=sAj zVFjAfeg8Foz4GYsO)W25SU+d_Z+L4$Goouh85U-yb}-nX34w7&-)V@q`OmHqMnjS= z_IMlHt6#t{AUn;7y{l1r00Dr&NobW@*qwlk9D&qZ>5vp(@_*#fh6nB?kItLLG@in7 z58Iq;936cgpgnf(rSE7-Q9kc?>_uw5pnC1b&<|r?40M5CP?2Z)C243I%Xvl69Tqa+ zV3po%fH|Cz$>X!l-;(@j0$GQZH$$r$H*;5wrVVOH^mHAk3e~&_@(v9#Yknd(D3349 z%Im>9&%|5{&j)A4f@CcRYtJ%=U$ z7(+@+a^aE(RQZ;%T!fTq7Hsz@??HAbc|VQe#K*+^QypPL9iea)xZ3o>`YgA)NXGPE z8)0237p@)h??E$1c@|-u^xqZ$waIQU7wb=s>F=Cl=p5B8S-px~P^bFso7aU*H{rNB z$}jv|-?+i?j!S!Wm18J#@J-1G(?Oo1i#B0Pvlz~XXS>snRIKNd*LO?YLLk=ZK9QbP z!4cvHGc1mGX)xyF9{Owlk(@-W$z;CjcOG40jDWtil=!^5N>Pb$*x6dqdXfBjl} zlZVR;uPw&*kkT~j&;a1b>HZWJ%SkUT(hnLkuqWcy(`!^mOmLnKK;1H7`X4o%Q+ja> zW!iZTgaS|9<16OPJ?i0-lACW=^8{4`s`Pd{7s{#Cbx3KOf!K>|YUJAu;+Ijp&(RWhRu0aL)lu1e zb$2P$UR-dcBH9(0zV574rH^&UoD)0mCv?tQJmMd4+7}Yio3`W`;va*`c0KgE$xH3K z{wnd%y&2CF0&bb2B^--YHlNML&+1GVJha>o0;+!OT^`515fkD!k-%^_?+*zk29e3} zqi>r&x}B5pVgWGvTT=_xaO`48kw-O-nU=nlFctJ?j$E0U%n*jkQb*`KUWv}YDH*6< zLYY*=g4^Vs_^tlf+z%-PUfF~ly3H7G2>>da2OET9X{?subLk`HpDndS7t50=TSe3N zdQy@1ECV*!)=Z4kIz+Yv_N7!RNW*%fj)Ay{5|3Zu6>XGpum=i9an@xdhX1R+D%ItX zHi=2Vq~_(h_}9HFi_pIX#c2I|swf=Ip+nLC?&OSH{>KNJbU)dYz(`F`B1Sm*p%GJ- zr=+fe&}k=)tP2sh0C`e*Tkd(m88!X;vRaS1%?Zw;&J!eQ#4Gnj43#3hCF?g0o5JLr z)wh=ZavSZ*PKz#R@-vsXl`nO#TGlfLss|J0_cQ2^$=8+D*no#u(Bxd=UEN z0Jd-{$?DUdktQ#A)_*$$Eq_Z-Szf;b{ABSQo17>m6Apz&A%i(b+V4&R1yr;ONSHb! zMdLjUWhKC@B?p;(W(%k$ra;1weIXM1goOB=mYKn!dSl}QB@NArSx19Zy`T#mr!bBL zwaJQ8=S{1M9-!v6Eo!Jh!5TI03r9b<>LHm~r7It2Ea$N>u~Sp#j51g0%lPwSe~Mfp z79M?${*KYC0zxf|v!#l)<=|xtp~hu?UmV^w335V|f>`6Iq)lP*GGelpdO@^lnaN>R z2=u(ZBlJECvSSwmz2WH1*VU?INPeeeGYZ-*gejDyw3nNWjLYT_vRg}fuo~3h85`9~ z-6|k|)X0iJWd&V|H4qXN{UEy)IrW1bQg`3$A-w8+)3>g7a_sm1MI08m3Go0HZw&q~ za*8IAKtFmnuppw2m6tVugLo%9K(NEm`-i5Ax&6Yz)2zHaVLIO7BZ0>y6xA$d6BboG zE*6_4oiT7giu%xH%e5>lgE!Fl`@(oI(^TH?{^kCB1@?iQUt2mr>IdwE?(WjXP`7>_nNnr~X4)Z`Z-yuAiK+39Ho_X#eIL5ei zC1;N-C+bZA-r%!GzzyxzcXD70-Qv7pdhYZ@=P-KC9;$tkR~$x!nJ3B#7N(UIq?W=; zx*Ez*PpfqXx`6S33Aiq8PG&`sM^#|SBUDfLy*B-nUkQ7U;sH7A^$kwOf|ziSoJK#P zm$oyFl%i%!QS{*ZzoEjN>AASQ3SqEtUqZf{uDGevaIDCobNR_lDW_sqzE)xg+@fR#-j57sl%>V2#mRp~a~ja*`aEL;OrcyW zdWbiI-wF>j9MC4iDJ{a9n)pYVkHyH^biid|w$!GT&ZBJ#bnmvu5Jfkqmg@|JdN&5} zND}_bh37(K?zc^Xl=_m80Y{j<>d2%f}?9S;n)0^CB`sm$#+IcTxIFm0| z^zyAxYJO35bNAmk;VW}B)KGKJh+HY20-)LOQ_iISCwz38|dR@`SU)> z?3hOLWUct|XM|LDcNO|&ku$hOB_~sP17{wt;yI9wNtRsuD zy*HlB@S-GPY@}~9TX3F`C$m+8+aH%mx2dRs^oRV9qBC)1`v2qj?#*0do6ObboHJKQ zWSg05UC4(%qasB1nz>TG5u_A)GHFR3i*`lYDy_zY3g+X%*Z?r+gx6PD>O4VM zcF2Q!>X|yiwgdgs-XG`@&HsQ_W*Ht=#Y$iTCE2?i8Ubbr;FPRgOXkT=!I)p>+AvP) zTM)**0}Bg=h&7O9g(qNxA;;grx=YXxuY=4w3*EoAv7$Z`AljRLdXU}xpR zvH=dMhfYE3u*mnSkH~T!U5ddJ^!XpV?z^A*dJ*7qEoGt3GTAsWBHNDIqoRvf($g{t z=L2;~UdRr)gbe5{BZ!U=q&BT`A(L{}0Y6=Ci*y6QKFHwQtPHqN7BgIuB!XpixUJS5 zv<6{Lko-Ac)dr?yCEY+V+f4N)K+%23*(~OAlB_Z*(WXeY>DPg1F#6kdJ*tVBX)Sf& ztm^6Avc7o4p6|A5b#!Q_1^m665gTqhud_eN|CgGH1qskNtT>Ae$c3V3y|BAWxc!Yx zk98%h9av*`#>j$<0UN%Teae&tM}4D~ZvcWq(vwMYl9S5p8JYc!BI3CO8LbW%2N^Iv3RMk@ z&jF!rXVjPB`NZzD;vG>7A%#6?$0s?;erW9^#og$DbIUSIkupdoES?#T#_mWnWT?3-NU;15sHF?* zlm|4_iFI;M!B~ll%B*$#3eR1GwHr_(Z&sw42Eg=Xqaz%dA)b19CHA_#@D+l><@E!% z-c|e~?kF@?@C6-F>wvq?U>SzneKqd22hjf#KnA9}fh6Frqa64*S#(1&m2@^GOg7N- zY$wn~r^}*AQ(b_7y|mDCaW^f>aA^oZ9VOU2s~6eADuDi=6r~2ujS*> zCZuh}+XhU_mR*ncVj?_I!PA{V?hsE`j_J~j#ZgaV=oBCkm24!#c|R+Y4cI%{l6Cf& z%weKBgzX}B5m52txI6)IrfaXeiv_(-1~!&@ z#iW;l2S8bqspIC{2im(N1(0}wRU8W*xf1d&8GX>vq*T`2kbGb-De@cnxCQ_o#tez_ zrjEYb*|_ePR*d%gj%T~=YlY>UeJ=`$XBPVHhCpi5bXh6$w44IBtS6KMUxyFzNS01U z)Lb$2wZ|9VW?&_c`XWiym)&#YSfFcPrHh0BVuh9rBA(L&M;?>KkgIE%FVMRAmqatg;eR6KMu+RX9+_nb_wq14q9c%JJ*#o;s zbX8~M6wp(kwX0;;^I!P0<%t6~ly{k!n?=YO1gCjou{buE2k0_iH->hCyG-=-f%fNN zxAtAmJFS6DX2Dv!c5e*;m`p}%mw_@0Ubi9p&?cHViRTIOcHuICBEY8 z{@1!Yr8ARfGLFPjIx*HU-hxNKW*ozM**5vE*r?B$|jk}uZNG+ViF zwLUod1PeM{lsc=f#KtQ@?@u3Q&X^7>E<(=sH9Hu8$N6xv-2sXx zKyep>j~c?X=;Xdb%&ylm5t8iG@uN{&wpp*&_eL-sc=(gUo@l*|5$S0oqYeeVluOn; zQS3qX-*x~(3?bQo_<@&?x2}WRmSIs6JNYdb<$34iQg7ci zH%oc=@ZYjYFbuPKklZBGHJh!yht{?^L%bQu7=l=Tld)-abqu|X>A2usBD#Ce!^(wF zYEV~DItB@N609%lde}r(i&c%Qwc91bQS7wvT$^~IEsG=>+F#b(Q>Q6%D{z{kqc24l zLECJp)kjPjXEe2t^YgwPs+<277+o=cjrZ>KRW@Rd-UScYG+>q@l`FCic(S0@XsCvk zqO!{Y%hab~>X^kO|3~7Z7E-F?$IWRRzi!T0Kw*;hRFYALg~ohfWl%@|FT9Sfj&q0_ zDn-E=WavuP7)nQL7-P)7TG*-~+yU;3KX9`P$1XfkA#wb~ebDFJfvnn|A`f(czrFC? z@DXre`7!Ih~t94~+?xvhqiOnw`U^+l08a6HU0hr$g_U_cG@2q; zmWXCmYHKY`y9df%OLqVuj*<_oTvr}3BYv)f#?M^0i1$a?qP0^{8fewc9?cHn zJ`WJ)S~g5;!SZ+}&OnG&SlTK7qG;MNoqTP6I$+mdnTKVMWi^-%22%jYBw+MX;qH4G z@UyrrwOt;vJQg4jKrs~0$Rhofh8RB9<9D3E1+n8SztlyiPP`-$5D=m4)o%w|{W}|l zee^mzz2DM#p}N8zu8}w*v{2UW;spn_6n6kZ3VL&%Cd$|A@KUKu3uL38o`82TSYyh) z4^C?Qx^kqiKN{hoT(!M-Bj3Mc?>a!fP%;Ow03ZzfQM08=TJdRkE?lv*bR+G?EC)O= zL`)VGY7vy+WaaV@&PEINDcQZ`^%BmuDDvOw{((-BmJ-X)aM}a5tu9S}_|m}LaZh3* zrpo|3reZb+bQY_{FanzLPdNzRoc}5fllQw79%G8wt5Ov8`|-WDAdu4c{@ix^=`lHJ zS%3XKdRJuW{-`VMu1BeLn?^trhNqjTBZZjt?~(mZDSrZ%1-(BJY&V94eDJ>|LD?NQ_E%6;Y36JOa99JM!5FetnE^x2hx32 zM}BmXUYV}EMh9x|c2yda9-A58n02T=6R;QY8VTEu9ZAm3=TJKCp%lt;^7G|2qzX?P z?VTsa^Hl?y6l9RCwM=B~d*+;IFp@-_EUdvOAB@@N5w5vEo43f@Zs* z4yGOY58h=lFYlnH1gTV2wdw(Acfpv^0+Z}W!9@u<;<*^(0tQ=+JTdEm&;1Xn`Vnsc z5t3_Vn*auy=^+Y>d+%;}%p=M5WB7FWpUjY27*FSI{>r7HhIcyUz6xFDZjPafnx&;4aXh%M>Y^)hk;w;Z%$OA9#6&Pla-tX4SHGE5>bHu$B*Fi`%*a^ufk!J={kT@a`3e6(s&T zM{zZwJwra(0{BE{=idhf*137hNUPC492xWP?~iAv8G3QG%9>_~*pb&+d*gaPG~J53 z^?`IrS+yCyN)mM_(=HL?nSkFf|uFPg4RMqV>UaxBZEz3bn5eo>n0Dxm5s(g#m zg@fl(hi@_jpqHI^%~2smBE${$=-pv0SHi&0#c%p|y&g%L4+|t22$^So+SV8m3#_1O zT-C$`?zaGBWcp-ex6Wq3e%n689V3Cy#SWS+HJqp< zH$R^xj)O|D(B5~}P@0d1IEV3rZ?k^R!wB@L(~a8caGWacK@vLyxCEkSp)(1k3#jc8 zZpqCEKIg{Oz;~X2jYvU8WB7i!`2tuTuw&qdHWR^(bi@c)unP=vrZ8_9Q)oOe^;zBK zXp949Nv&GDnsfN@f=a2eC&)4>#xP$-4Q*-sprVb0LZ7JmAUkMok?$DBC8o1jBV36X zxOhTH;hVzauz3fKJza_y)@4IUZMD>Wu=R4W4HK+a+#|xOM389rx_vcqZiQ^ZQI~Q0 zarJcC!(G+e!;cedGT`p>0OKYJWBr#!e)yvUF=ou@ZLP*BtDc%aiB#i{G|5DOGi)xzQ)4&vGLY_%bK zPvwZutRM^S9jW`{Phrm-D;wLkw9^*h>x5yurQ@NeBEF5fY@c8v$Qdyr%pfkh3wPFA z-8NV@qlICZX@OXzVCEY#B}Ou@KzT;K7QnB~c3HU|Y@@(gA!c^Iv})~2t#ua;_a%~4 z!(&Rbe#7rlOC40?sE1>S!{65`_kNP}v0>|$@Yh6U0WY3D?Ph&qqbf?=&R#zf2P5$C zKMm&=D61MpIY7mX6YvI!#{9#=k>}Iq5B^wZfv5;(CWG0TS7|Kel$uk!*i~}aon{@# zzNegVB2!jEgCtP_Qi=rJ3B`~rb?xD z8DxKYi+nNAX8nGd*}5lw4UkP>y|;LT$zZR~mn{Q6)rESTc|t_vt{ScH#K*M+4hv{^ zGe+n?^qT>%40jICHZ+zCy`T-ZuJ2JftgD-*iO*shSw~T8_q;wxKOSNP{34y`J2v7X zr_O&&WNU1A2#|lH+c#o&**wayRpU7;h&b}aru6HRuDrgv({l>}g}tnlN0&o0O2b_v zhR>cTNeGl#;7E(tbCTAE?_o;_!eg%$-+X^Olx2vw$%@QJ)`XZmnkNVxX()%{UppF6 z9LeJ!1qx`15U=PAN^_JH>QTqO3e#y_hXaVLwG(KC!NY_7G0$+XXZeK|9|>L-=@&Is zm&V+WbRB!z7FepKyyI*lf9aP62>(ad-3lQ;N_dFY zTHoaZxS-uVdsi2vBG_{`_bt*ja3Xb^H;Iq1f2t~VS>J5|V~UEoVC?Il1*zkV&@{em%a6L#fRsbPolUh++nK*e<~euR8Y;W>ox%m@5&b>Z=cmMU zzjt5FKDipNyW=$QOC0G&&REsseGeYb{5*0wF!9z79NC0qoz)QJ@#Q-9X0e2B+%Cmn z{FiuN@8{jxz-ZFyC*e^QkPlmrZ(IVUX%o5cin1hF)t)5f0oJRbjZm<5GUZhSL z)lKyE*cYAfm;9|4kXyc>`KM=JPo1`8kuksv$y~Il;pO0-=YV5&Nz_@}PMN4`ISdcn!ry-+vYlQW2y0caO}S*WX%mv|C#-J}xnLRWH-!NK-OqcPa@ z4pMnk0WQqauvfZPRfd&wZpS=9^zm*sJLe}z>;(ZJSZ@_=>~wjaM*)g`E6TgfkS7uf z&VH)-MIv*0`i1;ky(}A0u~cp{_9MFNFXC3w9?oSvyLg0zB!atGI$e&EgM#)wnEV(| z(--UFCQ7D?VdBf8w}M=v*a$PPf~T$BL%2jKyOnTPhrnx3C9$vFf#=+%x8-kLlQ>Nf z7VRlp1527#lh?sYQRzbIklfA45B4!*L*79+X|tz2U}33^df+>vgh8Y zeB(iF+V4;$der?Qzk%uz(v3fPVdEIl)V?kG%F{aFSBjO=@ye&`tktTAnoG_4wuH4D-LgbP$2X?h-52N~vS0F0Rs0K|smoONrk2TM`Z< zCy;^?Q7P4Z#OBYU!|Q9-1xw1^$we`LZDLCX?N$Ma_vE(}oyzECUr}WW=@*+Jqo+r3 z%;NlFA-ZkGX){lod_u9Wn0B#}oZWnPrD*SnMSCJ-pM?&s>M(+)e!nmq@Oz#`3|o3wn1jNEZ*> zBT^WyHt*d#`eFV zj`IV@dI@yKrYqRa{Z~gn+MB+hhr4g(esan|xH734hWr`qG`Mt*h}X*Mqn8DaW5ve@ z8WgqWgkpDG>Z{$~%;nB!jP>9&qSVJ;lopbmwq<#=-i7<05}ncNNWV7Oia}c6loLRB zM=sWhnd9-|&zbL=gCz|f+HYNZfp#xLD@}4i2F!E^H4p49xsWvLrG|4B80;zC(`){tFa6{LjO;MQuGML@TZ3fIO(Es!)VH5j*Y}i>2p*^nM&R!cP z4AGnT_di^pwzqS61Jbz}W+}JitQRy|uxX9n?Pl}+Nz*31fUDS6>i#cd8rOKs_@Qxp z%!L>6x$%CZH|Bbj$|RZ-g--<|FqEWO4_B87F6*^%K8)BWG(OgMNeL*aihuT;?0n17 zP5j2Q6V_g{V5i_K=cF6m%ao@4uXxv}+a38aBJoduX763!A?xP&Wl`?A&C=&` z)=zt$kLJ|)g}x%c9B|UpAK^<(^Z^HEXLf-97GzqI7>!p`3ly6(CTcdHy@WifBdPqf zINRHKCXXc%blBsJve%_M`Ws}m38g1>4aKF&SS@2!M6OZ&Ut z$(^|KTwUWO>fcLV^~>o#;QMI*yxU&0FME$UBI4iG^Yw}dF~kceon-`E3zDAdcc)#- z0cp@zlR;B2K%>vl#zngsCx^}nU8a@%x)*F`FFm|FAX%+==h0s8mxy;SP&Lo}yo4|7 zgG$PVXP-=FLo8lSk5%39d#e9@9vgHL>-YZ0oXc)HS{(R0q_sQ!~H`kY`Z$=(I zpR;@QA(H|2nWN4;CT+jcE-#%4udDA=jWtY>6-I8lc;ziQL*CCz{bcr~99+5)$fJ^l zrZ32i&&4H!EndA;oFOKtBQ6jGge&Y^qB`%rkQ zOW35wJC|1yn&FRaPX#{-dNq5w+HCG&LF~Of$HM=Zz8$u>qu*)W*cSTZiVaW6uCVd# ztIau|r|*2r-$%E*(YxQrJbfSg{(aof_eWqKSQGCVK7rA45s$qO+I)HS-TOUytISev zX71m+((Neo{b$eKh4}CLaPs|!Q$IhPhAp$@mN}ZsT+LwZtV2i5mc3jfPP{%~9rxk< zKk4Ld`=;s-Dfc6!CYBRa7BYV>pMkCLB9mz0ykk+1L?dLxTArso|e$){%Bdhumc9sS^Bj}L4g9ZPtUSqySm4)hdvt5IgepK-C94e z(YP-}ZmRe9Tg=4k{hH0gKc|6aPI{W}P)+;C=3;~fF<-QcUTfOtb?20tIvL=eAVuSj ze^$TDxqgC{EgJYQ4hw4h2ESaK2#{(B_Ar=xjQ!G)6^F_?@Il%0;`!IF9`^LVKYt-< zH2kT1-`}@47-l=LPpcEJhLuYx@1`sTE=sU0tecB?r$L__;Lc4;_mQ`y8cL{t$HFa9Gkpuj|W& z65epu!}sM5NqVIE*&{3bwa&~(w^hP54>&%heaunM{nr0={}xH?Y=B<)n~x4>zD;;t z@(8`t;QsS|IP!}0^6tm(-Rs;d5(O6j{nM4qZLiWT&n3nJB;+f<>FTGPyA_oA-zb3< zA=(#93q{-{-rv68MMGkg>;0)uqn9M??HckO%!Y4PQ)FvBFW$iVg6I->03aC!s!(yZ z&Ht6$`rrEi^zk3Q5ixx#i^;v?Ie~+1WC%49qG{-Z%X)FksQ=rSrw>cT-9OX+>GNsqPiq%!PbK9YX(~`4?QnK>H;=M(&u}LtnY`{QIOofwdq8W08p*K zk?9!M`fg_Zp(eOI@ps{F*;=sf{KGr>6MxhJ>df@^gwHG`?h_{A!P_&~+H@lUTWk_+ zIV-l%!X_`&dmov8n)Tx<{I6qsu*a#7$@_qwZctjlSJz|P4uXsTHu#|5xb{SR4^G8= z9+WLHrE%U7isJrD{(38QwQv$aJCvv01O1UAy`$|5^gj~T*$CzaxEBm`x8`Vf)XEwR z4Km|&wF|zS3ZW?$r)_4_3^Uw&pn8TsM8fI?{7%ek^u+88KtBux{E^f z48wG3w3cGh6+=vnwQY=5KY6E0<|3_GWIRYjFMFu*?edEGkVp`Gf3bnF7YR6q)6{xL zDOBn;csE+UD|DhQ#pJ`pd7E@8{Z!MH>8AY`U9Y4jy(&A0Ly4%4IOBU#P4t55+#9>c zkjTiMzLF$0|Iv2N`P(Bn{TCKDnKGO!?!?iLdLpCQ(B^c|qPFhXR<=F1pgc*jsga$Y z>D}w~##j2>j}w1ZutY^Beg8gN+;YMj}BHF}1PE&lFx8AM2_x4*PQCiRJ( zVFLvog5%d_tDxy?C5{K;GGvY($3Ec_V_9rqt0a>j3%=j`e$_d`eeH2~Y*-fd9JRuo13kKAseihEb%}oHl(Fd&TU*Occ*yQYS9>O(EX>ZH>E32V=sTkv_tDvsL#1U?p>?B=1^~HbAp(R zfy)EXP+-<>ZRnr+3?FZBq3OFqirK#d&Jry5V4sc8XP;b4MXv8^EKwLF9ZxpnW$M*b zGpAKLI;K>h?EV6|^9j~ZYtr{i2YJA?_!FULB=p7BZ1e=v!z}GskhtDz(&TWq-lkV$ zxON;U<&pXuhe}ZNTi!wVUVy@deyz#ch#aEqvzicPw`y~5?BC5A=ds7NcsfoDG*%#P zV6kG;_~{krSw~9#@>xG^pL4ID?ZNr@X&&e=-Im!77lUYti!o6n0tk|jD;cSSmRZiJ zF_>n{)p8|A61=Z?C6(yBQU{9+*Tyaz^YyKhDNvsN7yts~h1rXNyq~uG(l9qZhw%~QQP>e4Q zBi$Ca1vFNi`>g{$iyD+(2){>R2_VG`2uw8{YW3YY8;EAqD;;{UchTrf>YUn6ba%GB zfQ;>^;NrvTkeZmKos=Gkp4URIGmD@v&apEvGs<+SP*qdYPS>_-$ff)kAyfI#(Pbj8 zbr1mGivsB$??{(N5lz%SDaz2Mc(zINvS;PVL7<%U#(5#8H+dpk!kzzj9^i13KSXQ- z*sTjdaQ+@qPQO>YIJur~|ErR_`((VBT=QbN`a=5dBf+^QwV6PQA#{i9B;s%@1tCI_ zloEtHSp$^w)?8byhdC3bl@j85$r(CUgODr)#p$eJgLb={MoE5lMI1@dQ67+m{3mV` zFTbSiyQ5!fcd0Rz4VAN8;}J*dQ$!Xkn7J$zc`XM~!^&Kxks$`0#}#TP28qNU-P(!u zwq21BG|3J~$X}G}L}%@afHTJZzLKco9S&6Xft?0dM;=ZFn_nYbQlFk56s zPCu`lgfZX_sj()Nkt|`AN^y`TPM{LkjdsWy0YLBEw4+0flf%Mu;*OF2fgI3Mw?1^E5SBkw`aBE4Lzm~C7Km>mxjPy^C_~)JdVR3XO2<1k^1Z?RZ){E0+9f!;PrcAQgciM zp(mnnh2XXGa?Cs3&^=@Rw1>*{DV}a~2<8N8K>xK4fFKyagl+S(-innp3~*rdc{fcQ z1?rnt31hJ{2~GK6zxnY5?df5R9_NXBX+5T#2~|_?-m9L(B}DL{XnI2N%QUUJX)B!_ zVZlgq7Fqmm*svI_PD9Qx7=3eo;O-jV8 zR{>Cqs3;f7g6+~O1l=T2<1Gn!9h+ZeLUg}t7Mt^Jk8Mx3v{f|j58<7Wz67$Z(X_SP z&OJ9%fH)mngVA(l%Y`LKV2{P#Ugq~N?wQhNYVC^ic(W1&1uHx>pyIWVbuha`O!3pv zPyNr_h3~n|rUF}o?u!+wghkmVe48R8uFI7rT)eQUFRr@sp9xD?2kLo5QRvKZM{T!T zd%4;wLWtb_f*kQNT4utH;cXST_`ZJ-TfzKXfD%^IEr2+>%>IIT!Azh4DDObXTU0GU z$JaizTkhb=-0$&sk{N?kq2^*8 zE6xp&-E*JfC_@sy7xo}3|IOJjWw)q7jj7ywfMZBXx;?#H8jvE8I29-rzEDSFlMPs$MD+t|%xqK! z`b-9nn-9{Q) z>Oo>4_ej2kx1j*{8>CmtxqBVv3zc-t-`0eBJfsVqmP%q1xn8748&=sm9wf2B!u{_& zL>9ZHtK0;+tE8%Gj_%xezA0B%zVI=mOan3UR4)r(yYMLOiK0_;D862Xi5XRi;>(5ZIDEi`_SJYSmixI#o z1RNn7xS1}m2a=U%1DR|ri=v3JyQ~MoGXcN~8HrT^1R{`}BZ&YGFu8ot<%Nuo0_OeM z`FPcc1f35FpgKOlkt`32+qCs~HA1S!G?oJOSmfL}n(cT#zTyms&e^uppO=uvj@k<< zhH0CFv>7{wTNXKz`nwbkMtNnFE+j5L*j1FF7YLdvl?&aY01ecN8TO`0P2{`TU0HxX z+NKc94Io%zk`iT$EMaSk1h_{?u7X7Gmi9qi3|wIv1O|NwyTmG{D7axm+~%YrtC#KETx-c^2q4Z4Vd$s!@HL+Ij8Qd^>U3uvlcgJLlh1FEarrnuzN zfLdU=lOV&o81h_fj8hzOb@ee$S_l=A7h{6R#MPijsHz?f5q`KVt>9)RjU9Ae~fZX19=%k zxx?sFiJ;OgTd~1u8)Cg?%Og#ta?O-LM<4hpb!_S909p~%U#bQ05axXVGa}#p@JQ?1qL!0Xc7tSrU{AR5Gw>Q3`MUQa99=waLTwp0Srta ziR*!M1Ohghdh8z<&=I3Zcc*U-N;)3AT>KzQo=_!YTP%o3N8a%JUCS)=5mk=1jhD=yf6t&_xD+H!Di>i~8R z4qm!Mvv`o*mvNew;tn7&ndyD{N=?zf%&E2M%sJ-Hvj6gGI&(#Q; z8AVSn*-eNvhe2V1T(9S1*|Q^|$#Oh~M10R6E}yF#BbQhUsW7X*T0V?eE6Gmt8}iEh zikuE4MCJm-OkB_0!^UcT#}R*Gw*Q^JKOA45&+zxN3kr<9wlXpqLzE#swK4o$&R%o? zIMQN3S_+V|bw%FqnT)vsY%<*~KKn&Nc~(rmLQ|%c2l9LeBxbPXWxfMkKz17eeRC?y zNptOJiv6@iGZ~-^Bg$3p=RDK=;`HOo_f5Ze=+Hyb^LsA$Yxg*c&7ow=P{0;aA4t=| zl!>O3KrDcw11RmYFCpyRemYx?2a2q6x-z@tM!p+})G0Pgg`1Kdn321mX2`m*?UQvy zS5a-k2>@h99Fh;zX_Mbg;AnX@UD`|s&K^Kx*u2Jgg=D?qCN5H4@XLj&Q$d2z4=FTn zlPBLu$Tw*TC{EnY6KRi(WZ1_iFsd%C0!A2yd&};Eg!)) zUUbIezM8r=v`2V~ULQMyE43QH@i|jvz#U4Z*5@Oxm5n@d9p_t~e_I5^&A!|`C6*RK zHfX@LYpOno%1}$n2hHd^Cjx5kXnQjTphu{j0b&$3`yIa(Uk@*L@HXpb>s@Cp4guvY zRGQ0#6#n?vpZE2QNsCcsGN#jR9snY80LK(gt{9)FwWoi%Ux4vQVP%VK4k~Yz<94ie z85{(@u7zh>+@B-@@WUXv>(yz7vS)}RmJt_q=&+VkFViAr|Lg$&=&)*q&s(%7YOJT7Fkj+g7PQ_Wu(j#>gApSHow z*W+_;xdp`%jfS7GZh*(<2bZoxbpO)L`Xgi6h+}TKv=jM%yGJxYDz9;`a14aAUg?Vo z-Ur8q7<|jEV9Drs_(-yZHdTS%1BrkBg??plyaJ>+{|Z|TRKjEa|!YkgU1igH^h1gWt31psy;)SpwYBHlyBch)MQ-D{J9HeR8D)G zsaVH}$(7fn<*YLWdk;hQo(H~Vgh6ArR}=@)sX35?CqUu0gCl$KSko0ihG87fB3*qZ zba#f$@sv>^2-iUUIr%VX=u3WN>9H;C5i5}Efgzm-E0ck$Tt0nw$%9mnZklPzkQ=nq z?tx?V0X5CV*LvS=Ryn&i8$io=g)OkE>Bm$LJFf!vp9U)2?dj(I;yqnHa0DO|-%CbB zR{{FGcO}_kqSamOcf>6ptI7bi@fe5O9d8HHP+tC*CGJ9#N)LotX|Nijr&C8SwA z^4#@Bv@G=ZjjH5JI*9{c6WI?|#=SHC9_jrP+?jFJN!zC`W0$NNr+a0wV!SAD1eeG~ zWza96E}f8&`4?>bWrNih2Y~U}V%y;w!prk2R8nL#VBjz#NqH7giR`XM9rw-~3U&I8 zYm;V?5UXy#gGqyyGKpxCeA~eGc@1F)Xc3Tpr}(0q!~5tYdDES621l#e_knQ{kl;@E zX4W*@sW}de^bxmPJFZD@M7EFywb(~)%7|gOgH-^m(eE||P=@F8L+r1adH-fh{*`%? zg8#`*aDLx3VvpWHADd@uBe=T?XVk)65B2_d)C}6@`pR@v1)R8~H%Fek&Rp0}OAgE; z;|-4YK)`|IkI@+qS{55vh{y7y<9<4)z8CoX$RC-TZdk1>Y(lv5#}TM!ZQO3%&4we% z-!)7JFP!WN4i#b~8fR#qMSDMH&nEj&dW8Ll{#P|ND#LOIPKcX7-W5WxOsTo)76`fTzoOM9?RK@sf=h*+G5Yf;7 z8}S1*&MP?mg~`kueZ>2)!z`n9LzR?RTlY~8jYR6W)viewwMt3gw=N1m<%Arh(blJ# zW?MKEJ~kY(e$_TpO6%wR%Gw#BUFO;CkDT~zRNp|^(W7}yI$MXVOuT;E6iHM%D4~P! zxUuH0#nafab&GfHp1!B4yxa1mV;Lw%!m_g1;p?K9g-D`1Oqh9}0W0w8G;LvJP&}Jl z9aiPHbK}nbB9|f&_FA}O?K!%U#fF#MgE9$~J$lGIM;}g@LH$S)jT~g!C7~U*=iPKE z+VXhcV2^)vx>0czRlp7o&Jxq{Y4iwI<>XQ-W#uEgq9?KtnlA=*Ai*FQNFJGD9L%*n zDuJdahuo_pgQfDU;tv+xDaa9%ynd*5QrTA1T1=I0@rKDg;}wNuIjkwWiGv?`Zd}TI zKU2~w=AJeyK<;O<@%eN*>dU^5vWS5q%ArUm)Ec=C1R=ao8f8lR=%Gc1W)4%?rXeM< znJ`0ESm7(<=J*@iqt_%JZ5H5U+QOFba5$T#H$ALYME4lcsEnB&(W=Y$7}aiRn;zBe z9QPOz)y(yTYfzbfEk*7+Otz}`#4-_fTS7Pm78`(Qp;U+nn^$duaq#P!l-jNU=JHA3E?5LEfV6j_TiKajBefhc=P(%xP_$DdV}7}| zA%vY}PZy@=nlR&`c_r%=-!gV9_P)!%Z8R)&r9E;U4o%w3g~f~J zoCX2PbDl{?kiCM$r2H@-PaQMS9-K#OSZ}zrYNSBzto7F>yk7b6z_ypsD|j(O84gu? z8Mw%JD%p1XqX1LS@N7f-u{0jKsveW|%hRTF!jUw;<>hjFDems%@*KAzOgE41j3{TZ z!ZB)r%a%J%>`3S>F_Go!Oy_TD2g_JD->bc^pfyV>rB53MXNgRy+SGnIr)diJ+| z3C^DckiS9ROOB+VnS9Qt?P{rrGo`9-q$sg0M4QY5#Fn{5@YE1-6BG)zY{H@NR3&u~ z6h-w;sO&f_Q(r(5cbF`a*oGW;BEXPYL7p;|Db*55fgRc|p(d_nq(%!}SmR$@7tTaphf7!_00>shinEK5!g*vl1yIoW-l+VlC@0lnn zOuGJ7dA;I+x_7NaD+#GO`1Oo^pz3bwcCi$-VKvzjh3Os6)Sj5c*e8+18z>EgNYt%B zysX?gugob+iuu#p&H@T|SS_%#xV)%=a8(o{REs0z1qwJ4t$41FEG6B2j*CxrAGcxP z5q`s5Pbz72WPUz(e}V_L{i8rj_jq1_U{UVea31K`B+oUmp4C5@=lF9T8MPHEt$`|7 zH6u8c&kGOf9<9{jQMS$G#b>$rQXkq>&g034o~?iiN>gp_Ilj1bYqQc-R{&<8jA!}1 zw)alO_wT#4cBy%rL>kbHGN0ssk&!NP@?ucHw)g}?7!WBg6?NUDW4=WBUpA^Vs!SVy z-~ONa3Nfypj}|0Ugz;bLTatn1sB0y6m(}$a`IG1h)g4s3250-Ci8DlXb(MmbnhJF{ z-EnLaJ&UvJ|FfjOibJ?5-R7X zkA@mdiOEGaf~hF~cl2O|W){b*iv^Xrtx;BjU)0icL}-f^s^}0Q;Mxe=>rfBJ8$*%K zbI2rqH%xzSS}T@^%G8nz=9(sR$?B18kPiUyrD?mUY1poQp9K2hHjip-F539c5*7=z z3AjiSjL*+u@S0JEh&r+F_}pKyMkfvV8H(pU)vB4bVwI6xvF`aCd<7|<%p0kye|LdTjfzE*(ofQ$KDxm`fz_hiorevE#+=e< zHoXC#na~vmEVd^qOqZnT?l%c0ulLuVJzb55UD|LGz8*Y8`j2rL@8YoZb%qm&2y>mo-%nQ9I(CqSw&^^MMmc{ZTy?cmSo&qo1n$A=&?AgDKyq~O zkf_UQ59@A2X{dHz@!gZx;tI-;jU>w=Wz2tJeU0ocjVmuZn-A5WD2HCGymB}v@Z06@ z{YT4ey3h=l446eub=<+m!!`LN9kqC~^Sizlm0LuIb3!Xy=oC3QW@U$1l-zr{u1Ccg zKVTp&~K>ykeU@_^%T|%3F=4QIHqgNk zSPSO>lsk68wt(puLIwBKn7v&<|P<8D9H zt(&R(q^~drc+nPn?9S^n-G2JHeZP>xdn(2o(uUt9Sw*kM0-J(T=od~9THLMqlRxiBuH5W9j^SVY`mfpQ{lz})b^ew8Wv4z}q&i;4 zKYTX*P}GeaPWv>uylY0V>xa*OJKF(94i8ceMWD?VWOFa6_74H>T>|)C{2xW<;Scry z!13G5Eu6j1p6Se_)|H}UhxV8a#iVFAqwVAKk<+77txSwSQweEnI zLSDqtou(7!LE06tEBj~A^sleRQiec(Sh&WS8#Ipu$fvIkw30*9=w{(p z#|+k~5scZHufOM?LT!hBwtxNUmN4&O=qb_hZ*wQz-;40q{dYcYoo-{gJVMy1CR>~E zuh{GLiWHUSU32o@FqC{K>H-UNq&eKJ@4+_Tb(*;ECHyX<8OXk(ZlCuZO@A=siPCO=MzvKv zM@O!;I`o8}E*8@~T-HGd>tWE%$FFE*>T6)$V>Oq$k3jBzr{W&r)r-Y6jyAs)xFyK` zpW^5kf_LWNC9Zwf>oC^8m88GM|54~k6qoo-z(?{%^>2-0vfqhwJYNco3d^F|C5 zlJw?Y@YUW;{E8#~0;HT0$kx8t=}saBuPVMa{vqIqPxqlJS!TXAOzF@)Wzq6a_DfyC zJGPTNHx2)O7u_(%bm~1sk&61|n^v62VC9X0!~bF|n9@U(0@aHtV>Q&-YRHMzWa{zcJJ((Vtra;cxDT}I^!?8LLy zEMM~_ZFl-LCmV%cm-?s<hKOUF<5vJ-_bxo%=5>1KB}+J#Aw3eoRg7iIp67-|IEm=Wcs!$NugI?{QDu z62gaPr^;0BZc4GyytV&I#HmFc@1rITU%j=|*Jccp z<-@x?uFyTMpbu!LnqRxQKfTNPH{bI)!Wophr|5upYO3GyTYgpZw(3t6gDd~kvEn1f z{0e^V-!<-CIss{swAt~+rTUh6_z!p1_k);c?%u}x0_YBNthvV>2ftPrW*zX1obZiF zy&qR;^6=YEv5=bqb7mn6sm?bmT$auFi>W_!%ntoY9sY9DnOy0UGw$L1)3~U4ZC1+F zqGFHOgwN&9L#pOsnrUIWx5IWw8;Ydd2PSBtmEk_7;o_Y;f2X>zDngwrJZ*o4`S`km zKmgT05GWiV0vPN9PytntHUI!20*H``PFxNTC56@t;&U4w<>1x#+39pN4iw0m9GT&F zG!2yyZSxhKJDZ2gb-deyhC5rHRFT3T*EyRkR@PX^>>qdP4ehHVXQGr`y4psYoXYir zA9r0IyJTSUgV6ncr1&f1)exp4_?d_8|i-rfg7p#Zgb{Lzt;scgOo7wz}&Dx4Xu+yV9STvP=X|dDysC!U&J; z)N4W`7!P(2&0oMq|5}~%Ho_jx>H5tPZ(CQnz|4hU?7)MVi?2loaT*ei)L28x^0q3RJP!gllDo@uVBWftuZY^H0$)FF;kS$gc2bn1M(YMw%o9e!bs=r>oKBm*! z_AcMx_L7?GM=4je{N8T>IqC?=r-_075Lb*x7!^E~;QayKJG^8m?B%QD` zpovs9Ux=<>Rf;Hu4n3N>Wy_*NeekVO!5Qg&l{;^LT&$%J>RyU4Niz?1IHpHvl#G$@ zKYUTT^w0AnDkx{X)JZOx&3T~vV_w-^IwdgJYIIOg9P~?@?{4L@;aM&6ccE4Xt%SA=1Sws5bfuiXrvKLMXSe{i?NRqYmh_4VyuP3F$LB_(rJFvVwBZ znHo*0D_)23HK5NF&Fgpmd8?kL_lr9=oSeUWF-h@3`JthZ;We=_<^O#=eUi zcBS69YSs_0gdYDzYXTen*Jk5a=ha3&MI5|l-ygMGpd87NdWYRF{a28>nzr(7h?rCN zu5dfK_^KTGjr*ccKGCSM7ra}a>TBWu*yj~=w zU7vr9QLQz1Ui(^k>f@sy63sVW%E$28ZBV)1@17(v4V_xC_bX%9W$td`i1(xp_zl`u z$^Cqj{^8p1S-oIU%$FCC#MG=HZ=rZr?qYxyL}9Tt$C&bctK7(rPBQvq{ZfzR_y(%F zn~lvtZ{luWkYpk36vOUQJsq)xp;Sw{FU9wj`Meyo?fbRAU+(=@zSUT_P#SXLanb{J z^e2oJKIJ3O2|t9^tru6$;BVsc#J=6hA=(~3Gx`1z(nev3u-TvFV#i;sErETDrK*qL zYQ6dO$7F(NYeg^W2IcMzGizHN@I(+Oy)@UppP~E_0LAA%gjudK@p0wL+218sx2ud> z1svEeelPaKY99{qZ{P=y+J*!!y!UptVr$_SBtJxMK4cKG+N6H(k zIb?JrUXqKe&)ehs91Z1-wXGY- zu*d3nqcUx9zHU>HC6#Go_`RsgKqFxh5tBunri1H zi@u$|6{qEZ*j?$=xxczrG15+trPG~4#9B!{J{Z9<2-j@Zat<%6O5 zCl9m5JxL{qbNcY7tF zdt&#y(K;kD+8rMDl_mJhSrlzo7_Z@qlq}Oo4njD_g?T(_b^+wr39ZD(8Uf$!yty4OZIvO90U&o1( zTn*9W%$=;>;xl$#=THoLYk5^h!LRabD2RdPA19C42j$%Nf&0KL5c&oc3cvQL6r3 zMO+#P+&Vn8Q|o(cZFgo+@-(wY7_L}j))IQR(*>F#s}>jv5V!Qob}?oEy}KdG1io*~ zugj-y*T*<^79helLCTZ#xL`x0eD{>TC$;NC1PeoigIpyo|| z76pB(`(2fzL%$|bzt!u2FEDw|MtfVR_v}O(OEa3hlQ)`$)~G1~1-~&KVRGT+C2qQbzb(M&d}GcZGHji}eDZ;A{` z)Jp?m67)k19loS_dXO_))pmfd0vwTwTYtrlRe{<8xIrFvdBJ^kSCUpAe4Ph z`!rG$Zy~@9($eHxT=_X39>y~87d}ty#SNljQxic3f&)oIX$Bn3PagD8q#r)r&qs}E zup08X)eB5Wa@X=s1>)XN9TI(Tqa1R6SW<&shKY`|RF0Pg1HAB_@l=32*$Oz!!>;f^ zD2A&x5n{^rvsOLyaWFi}{G`zQpYIJfMf2_Ota9 zpclE=MIPp4H|99k2}KecV*?B6m?Z}GD(!3&2lHOUz`o`oLYq_H@uClE zhB^NfTc;lHZUP+RVUu?S9law}!o&U);2!dz>jI?K5ab~vWsQoBBg)t}rd1IkgPasM zM*JRQnKy#Wg|+Ohg(EgloDPM&Lo#RuQIbG<*_gN7X<9V55NIH38a&3^d_`(r)K( zF>r%qFv$#5BnWgpCT2?2P^aZ@ad6*Oacd0RUHWMQqS!Zfiin0%r#g63aaBCfBm;NN z2|{2XzO5n@{vfIVxHazK3XukcFUVi#VBD*GMO4^27imMqY;jAy8G(nnWzLi6%<$uI za~-q*@fv_#T*XPGCDXYW767{hK(c9-Y$BCK!kU%k3c1LmlfV`(<|H4(#z(BRVc0^* z0|xdJ7vsYrM{qHpxLDsj?AYpwI}^Y)BKnj7cLR9#=`ofR0RB^e#747|hsu6ZapA49 zxg@Mi!1jY?1*$?es7#s4U?8JJ;W9?-9G*)$w#q>@QE?H1pu<$m=Um)V67;)2@BsiP zMJ!LgbPSD$r$u=^U8SQm{0|dDkgw1d#1aubQp82B32<9Nv2Jz9PcE{SmcPM38?2_> z674?rlZM{I^SroUV}*_3O=_;wFj)c4$k?vm;o(T#0ykfML133`p6!12F@*O z51I>qLx){~h$#%mlnDK~RGizYPx4Y>xhZOP7~9IYV9Mhlk+^!2|HBB#VVxt$j%;f! zHT^2FHB#nXVeUi5g<}@Brep+*gb}RbSgYIS4+w!B-9p2%>6ixqBxe=YPzk)r!L;ze zHr(dRmB1DN_5pW$Qwe=@6($sj6~I;6TLGM<)4_cJQyQX3aH=xPF`bH=q{9xb;B8x5^Z)j*2`r|$BgLEM;^{tm+= zY$pKsK!9lHF$grot|)W_88%5T{#zlo0TA=%!5-$8A8H2Nag3%8TbsOZ?-StKdGI0z zB5?aaoqFLZ3447N!R4_D)Y26p%uav+Iy-0!%BQ(-0h1nSnsc2FwOd8364BZu@1j-2 zr~uc*KtwIWt{f<6vxMFuUNRj59k0Snb5K=6)E+W0gM)h|gl(^Prt$D;D)veRW|uC+ zH2@bW0Q?;Xo?5=tPD8e{!%cbEQ6UWUH!aaTw#osrXIku&u!&F#7gEu&4732yDdJ)N ze&`q@iFpVi0}L#S>vEHUJtu?&lf*uWXhn=PkdP$CxYSfg1ShG2kMl5}R^i8qd$sS# ztq~E)s_3JWrz}X$@!a~t4aCDGR1p`MP0N!0iP#{c50fH(ve5*#ucT8=y7`_I2BLO1 zR%{T^r-?pHg-zlw+1UWS*{Mfv=T8$M;9W&kB;3PQL=zDaC=TBs7I$)RCFI0(G2ArX zvIviRL(96U9#q7;R3t#2*Ivb+} z*KumH$ddJ94<*5 z&`QO%@*v(cl&$FFNfZaQP8ahhK_7B(iHr+Y6iDm`f2)pckq_`^51juRxJpBfI?+v* zZ3;9x|5gDpG_D*6aGZt>%1e)%02~BhuMz|0Xz15OFoufy#D&QT#e@Ke92L#yw#|Rg z*&cTf&$*)+h;{b8zZDO{Z#y+@a0mj#h5+%zmA%G5yAPg!#SZW2!LuXU-0&&IX3$l} zMMf%Sf$bX@iuV6tEUkmsqGKv^pc`~CDLmkS@O-5oXnH%r>E=*E3R#-MZ#tm1c+i1c zVPAc)KiL?`^TC-m(1*g<$`rAmfEpa4Vh4d##kM$MULvzw z57`I0n=q@iyCPctH2#7*-SO9xqFO-KSOKmk6cjyPMX?2g?C1C zpxJovW^HUH6s7Q=ixda?34qe1Jz-HnH@H}BHZXyU<#W@Oh~NV?^?{Egw5jOVj1dtD zqw=Kp;Q?j&duPAp#Wr%N=s^1{SzN2*C4IBH=Bafbc`NE^g7G4<@rRTTmzNc)HQ+Bkk^|Ex8|=ivg$G>0S^DGbnh1pRY0OjFLsJrV3hBG0Z!1UaENFJ}E(M;YVF#$`kO=^X zfj!Ov+#EnHabX0;lrg7y32Gc3WXPm|BL%y|C+#O+= zqx6YPn`i67MCwb)Iw~%g=H;BlM;#1bmm0UR(n;U;g_2+Vopz0$ov<@J{aLNcv-WB-eO9Qn zA<%|FWA!iGgkbwN0XD>Y+O*ig?Sim9aDk0!?FX$0_|gM{2>~mn50G~^`_P`?*l}#z zVJ$QdWUqLke)_+bfbggJx6ycvoG5S43Kg?D2{gdtj7}scdyNGiJ@%7?ZK9!?0JZhx zV9j>28lyY;M8rw*EW_;WXu;stV%U+F9}KwQg;&oDuGCgUj5_~&-@*5N^2DS_V|O=l zsl6W(dpIDb$OrW!+YjH3vmcXloJ_^o9>ZY3Ek{pVoeN8GzQLXXOaRX@unEMLCGOdq z0BlSK`me_PjPahTqWCMnvrR`z*2OM*KNHk(rG^5qtF-(z?pL&~)HELFZZ?Q%01V_F za{Cl61>mo;OVCJ$iV6Xa4 z5RRD2{DbCW4B!*i3?IR_8>1}tm{|s3k)6ZBx@*!b&~&R z`E7h@RmA=}aDU>b3rgSjw4*fK>4RA0zDruaaRz_B{qZOo<+e$!W~(0Do(r!&AnjQg zLqE71c71qNBog4B(j&rWhirb^>zm`gAd1&Xv$sNUGfoZ%IJoHaq77lF@dhq31h(sr z=jR=}?+UO6Nc_eRxhrI5#?>?4I$Vyh^=%6Z?cnw?J0Zl}&w?O(0KXu;_mt{SGkyNM zLs?k6AHBxWTpQ@owS)iuPBvtM_HMBd=XX8FXC*z1dZBS}e95cXxzaFldiZju)TCj>Dx{HPGtl`>9ce+gRnkfYz9O)2#QJV3r%PHuXRFp{8?$4RPDc?QeHbs+6rZ zj^#P@K0+D(d;|_KdN-hb*qcUnc`FA)V=wx!i|XgM`FttMemkBK6yrZ$LT6eEhGbw< zO+xu}<&z&(3=gPYQ}H9Zax;Q9dgp91W$92A)4d8e?m<#N5qn>~TY_Uz^1W~|l-uZG z1gaCcvGyXxzYPMV0i~4nDXiN(9kAybj@d-k&pm*?yMd$yxz>1Z)+%F z5a5`Np7qzyL@zC8-N3p4FNrN0Qa<4Lebb89GPgso{xK*-Cn>{(oj1t zT@bNI<@PFS9?gJ~6NY)7`#yD30roZPyrBxooePc1htiuvFGT2&FB3;{GYo1I<9eK# zy4#T!7Mjp)uVRqhf2qz^e$KAU;h#9f+G;+& z+da2Zft%(l(vx62676%>o2uVKw=x6?WGk)9UHt%o5`{cnb)WL zW=spNYjuslr8oTD@318| zJ*AfT`vX`VMbo4Y21@*~3u=!DUGz{Qmzt&TR;qd~h2|yuUqyi>^Rm}H$Dru#0#ML? zrqxm=VP|MB88xr-`=^#d zATTa|Y*Yf=>k=rT0<>?e6Oy^1WYc%O_maAnli%KygARHXIp2yG$+fbrJieb!)ktn%jsM z0S{5J`BS}K*tYAW+7s({Y%%LXfQeFcApQ+c*{9TCSHsN!IDf%HxpW~>x`49)m=XVEuH`Xto!g?1cMGRW{4b<-h{O%q}d3Wkddk7SSv<&>I zj*U`X@KEZiJ2&6zt;mG$S)KqI@S2pfytnrVnb}1;JZKeH?5`7_d99c7mwEoAyRk;>u%nivqFxfceDz1s1#FW5mUMVo^qA$ zD^`hY1#y>h6jB1@mJrXiEOLh|{7D%jh3|Hw0hmjME%_%*)hTEG`%?Du?;|Xt8kHQ2 zJ~=}kUHO+SYxp-uHvG1DVwQo)qo%y%>zs+iO_5E=ctlcp{bq!YW58o71!jC?joQ_g zh4r=g0XA!_YjxNQ}QxE-%h719Cs zXSm?$I-byO5-u%TTcLky%E4byc%l#l_vR1qIMvjt@hq#7a*Wq}vf6?n^F=@F} z%n;8c_@BBA!mF`*oU%c?GoOY3XtP8eu$0-BvFZuE0$?iu@lzxgX&ESHvBI=RMf0q; z#hdYbwPUH5eH2$l3#~YF5`MRkV4EE(1AMq@y?~!gW`cPz)D~3n_Gm?Sv^;&t=Z`tx>P{_wQy*ygY@5EsE4pF? zHQa1c*pP+BKbb;Xbk9k=>KR@E>E6}#u;2i@@RpuHw3NE`ecAF@@aQSPb)1tw$J z_?oO(VqqZHcuUs;@Evn{30^9Wuv5M97%z8(e^_)Oj?wOmig~!_-@#S2};wS+; z6amPz0Q%+ToY}@29oXeG3O`ld=f0T}=MP-@3%6c^`3Wu$|H+YBhq@3Dep5Gw7`Dn@ zg;`zzH6rFTzcgF5rHBjvYh1bAc1xE9W(t6%@B5=HwC0Ahjteho%LC=vP?r^WIw{Nf zZr+h4n9RE-i#E6gmEqv;fFc_eQ2~l1m|1^LVmFKG!ro`YJ+IS^r9#t|a@@yZoynRx zF16bos}w*E(G;HS59cp4>H{-j)O&i>#1tWLsE61n=|p8SVLD>c>tZ50IVXk)=}ET? zCBfEA3YRHjOi7!RDVX$SjOemFtPOoW&#p#z`;1Gf0cCx&q@On)TXqM9I~U*vlH zd}ST{;lW%02Y9sGu)-p9&scvfP5i;1!qS+FfFq4T(MT8x_Q3`Tqd#Hua-!CCGoxX8 zn{fRZn|%!Qr<>Y0dh*W7LO-_W97o7h+gR-#>v{7h$58-d=H>))m=8i3j&1NUH<!7kT$L8YGThLZoOrm zcFqelq*KPBOW$)$j~~rC!zWGAn7UKITDGU37a%}Pb1nIzk!2xht)1>IJ3<60${Igh zsg*HCWp9-KNxzas1j*y8GAw}Qxk@LE3OYuV!0kwcl!HfY$pOlapEQ74HR#1xx$-dzBBRD-I7*3$JJ z|7yvF&S@Q{B&YmOH{~U{;9#Ob3QbH701}Z@%O6}WlB9#=ESNW9D8RN-$Fv}WT8F)iyhA+yU^nu@a_u0eRLdw= zwL1fUg1wG@@nN}Fp$IX;1FGxz{L0N8KOsT4ErsXxz#D*am5=yd==T;9bWcA#u{v^HAhckjfPQ-0}1e#h(m#NH_By(#yYc;a6>b}qWM#-<+LEPkfs1l^+{? zIbcD&(MC{1l$2@A)8F%IXcX<&;IA#N4Ber>Zin*n5qO_gcpf9xw+=VA@HU?7JWo^Z z`|j<65zHQWRXC^BQiR^;UXs;USYcP_S#JLd-&&5a|6_pq{CDiClnj)d89^M`JEKrb znzZ_;uv}V5uf*tR#i=C&2&Lxw7a5Q0SIW`J80RNjb*COr zFD>2YUoHTR?NjoRg?}^`U3lsokw|*vd@!@Eyzs7<{*kK2qcROo4+2metgZ&H(Qo-$ z$fx_WhnIsyUTDKdCCjQ}#(BkgBYq^P?|<%vanLtIGt-+o`4PNE$MszgVqaVyC5^n9 zRtv9y=gl?fFf{F2 zm+mWwNdQ9mxn1RJ?a|@$M{8H19LVgo{*ZJ(kQs$Z7&D&54=2tk?Zg;m!H>PyD- zU(Z^4J=eWTsiBpx9U6na{66w_+4&4D92zd`M@BI?~QrBE^@|MtiHNAAG^rLn^6 ziB{WMncyb>FrVXdK3er|h6lziOFq`mp%!V-$j~e6fu2|cLiRWl<6kZ|%{6-|YrZ?@ zNMuGZ?cf*F;}Sg|nR7&pft}kcGxMNvr2txFW(+@5@r-AsRe<~Z*@}g?pH{{kYJ?*D z_Xqav$efF8*zw`S*1@i7&>?2tFv!=tr~bBdiiEd0&om9kqe1a-j0vGb0+ri z!6P>;UdDFLU&)@lYJPZ?_35~jJeSqk5v6tVs7~F#`4(Dn^J45kNZeq2+|ZfUs&fbL z4nDj0>*M`1E9qOGw8i4Iz*jjQ3#Ef`qsm8L7#I(aPkG{Ha zbo%KTAiH(2*DEs2g;#hjjFF zI_51M`<0Fp(eb!Mag{`gor#jxi3HC?sY8j<35hai6J<*h z^4CNaQ6dqSq^gpnwlhiHI!VJbN%K&WRzi~Y*(9BkB;AWidN-2vA0`<*PulS|$?$8^ zPEisGmu#exY`imBWMZ9c>X~eIDA_zAdDq!wi<0Es7uRkVe-3^Sax*01dG?V#M)6C} zo>@DtF199LLZf5M*l^>wu z{EWfm=4U@3Pm?ckpWguZNBuh(btKix5yyajZuc*_T&=5P8}sF5%E3!Z0XL$wUnfhb zXj{(CA86gbZ%#Md`jHNH8T_x&pad}0qxGP&;h=Cw)E-LN7?Hs?Ph79`{Uks z0-8dByQVVt&@*LqwoSl-tGf?H-i{8{&Of>nb$Q|~7xu}0U(>ukcrP*2Lx1eVpZV1T zzubSl^U;cOHBVK|n-)FF{uuu7$Lxb+F)A4rQ@{eh#pk!T&9dOVQ{W;}zHTmc>p9bM!K{Zs4Ww zmCdr?1|CcSm;3*N=>V0BfSp`510FCcAuaIO&lcj4&Q$sLf#0jpW6h2y`dhcz+@Mrj zT+Kj+E90cQVZLOOB=aiq((*BXT~2}+dwAR&#CBe=L#v(@YiW z#gC;C{Ig2c-1Vxt`Qbtl!R{`>o{6YdGTJ}2Ue@efZFNjk9Gfaz76bHrn4$vs=eGs#7#)(jq8XS>E|5OT@HE(mQHn zP*FyNn5Z04%yHg>?{6S-1-BGq%t4QQ7YF(r9QC?iw0(StYP64apJI5Z-;1)!s6Zaq zetk9We0Sm+zv$g))aKCGSqwVSSIHKp?rB(q8`;j-6&}NdA(sJ&j$83SbLNm^(RN$#;VB9 ztBlY;>V$;Q%_L-++`N1}uWcOwCJ2vLYR4M3ra+Yq6p;$-y}GnPO^cK}Y6q;OTI4uU zc?4SB9=RBeGa7TRE?=p3w*x5E&5++Xu7At%f{z+)RL9~6E$6Q*6eFd$P5nZGOH`En zob&90MS!Dz1y|}!GODvLA7ylGSZPnvz>e6csGl5HC3+-LEkI(z3oO@@9M4I5n^%bQ(6>e&;K1JOdO1`}KiRDfA9T#7>QbPz+gV7IjflSmBHpRsHJN zal^vu3Ns4V0ZJD(gvM+hAMy^REcC>_syu8lW*sl|%9wbo%dl@R?h#pzwsajic#$o* z&Np@D9jo6XY@k>6Teplg{uGOA$lQWj?C8GA@-`$DQUbbVItNyFi%d1V8ESc zXxqM4s)NIR{EV)hY4kAq?{=px@4y$01@LI&pqa*+)oSqk$x>bfSm}~f3ozXYf1kk3 zbnT`bZwYjTg4HRn=ynl1_IncK6U%j>us!b{Jv&C;Zz+q358AOu;_=Ht`L0)=Jcht^ z;tGkN1x$izLx-P@FTN^I1Mew9D%`PiXutuD!0+WNeIete`axkRlx$VtmtS4?ckAa{ z{imU2a(URjk9G89Hz)ah&Bm;_>lGA6*l4c+g;CiKt1qzWOOh8loshu}c50)nK;)Dl zH!%rl`J=pE`R(CmQqr7V71>UXKBQ<@$26*y9Z<*G01XwxP#6B@5F1yOc$GX%&pkW6 zHHSS07B?LUM1ZWkZuft^JHfCCO-(+%E#X1~B;|zm2^}7+b~BR`K?Dmop_(q~Y6oUV z{1wv>aLS0dwVMv#y4rseLX22vj(F`=m(&8Y&X zHcA3pqQKE7X%s+}*!r5Py(K-{QgnXU$)Ch$?)+Bb$eU!VsXo%#o7i$Oq~Q>Rm!Z)} zwl21tI|9+7GP8*Y=W2eUM_#dZBQ@Krzh&rGBT(6)H%~Y?U__vi7v)uvo)ONf2lucP z0IGRs(Uj|vd~umv#WzDguY(gw;yI2+`D<*}{SppD(`JFN&pEj+g9cKEs9vb@kLzY$ zaW2|z4BhL-v@(#roZpgf4U%n;^y4s9JqVh0ilqpcrdP?sG2nfPv~#&NNT9=vD=}5Z zej5e%u`yT9<3-QQ+YeIx>`b!!%)@ZELIrXz`msx^6nt_|e=_$`l@(y|KneFd%H!&Q zt{dgRX}w-$^D3Y;5g~#TLm8A{JYd|Jh3E=>93J%DogxIlgQ628AC84obkcB5fJkAq zAa_qUQ+#|eW8YJ9AC8e0(S0IWE*h}l%Bt9XKO3xEO@i)d8^lx~Alp#E+m@6|$1A#R zkgPhYvi*u$TV((SXA#yy!R1^eyF3eWn z#b$*{?FUh0!-`!t70>J)g5AOPo-`afqUfJq|2(8`IKeFKAaSw^C0l7z{k7ZDP{`$ci8iyjz?6065B@Z z>Vz7NLLk#U5Ab;1Iz` zxJ&tOl%h@Vjd1H#XXWPxgJjyo#7KNRmwfuD<3s-g>Wmqb-Ffw_A}4%FIyvvwQxU|5 zpcon2+h~>%zA|*Sx0aMhu{UE3%u!~HDDm~Y3ARLk5b(t{9;!?7G@YfuI0$#cOsovx z6+vol%n$b4R5>2rLfMx}2IShm&wC|n2P6{MrTvWJZBJTU;d3t@*kOD3mvRzX>pH?$ z?w%R16H7CwC$5l@{uE*mZI&>d2K8trdLs*Er3p^(=wfhZ=sFZ2MFhzTd$nVF2^s|m zgU3p4l&)S(ZiFL|PC+jPWmIHk%a#TyB2#7v3M3{znyK2&lyPx@&dHnkHkn10ISY8N z?*cLW$&0D9r*=-mP{1|=&L;YtMjcbvizhz!Y;MK@VCM!a6j2G!JjIUWEwYpeUJ#&c zMP+Kgt&o>51nSvQjtM-AmKFT_e5?OKcXW08K|b=8V)27yM`~bGbp!* zJ&b!_Bjv<%B@a>=t6fUa3V<|2F|9&foK^ap=h)fmOcM&6UK z!ejtQdkRTZ1o;zz^1^|T=)y@;vRsf28X^Lubn~rQSxzS~j$KvxiqjWWA!2l(q#Fyw zqiAzy3v|eK^^uHIW9M&W7gGE1<^b&83`<|P=qaysftEp`(9I60M<22_rO2Us?MxXK zRykG7UY)&;c3S<&ANHLLNWBo@-ZhcB!V6A?i|cS?V%&s;*;@<o9`3Q#>?#ynL2R|V?Hts1wHe)FYU9#ixr-#bSp#D0& zCnp;3lK`ijBI`G80V#k?Qx|wd;JT7R>MXMQXO2xBlQvk07ci#%UznZIYs)^8XeSBZ z@2FTYmQEkN!f?ttg#)5{tN#rh5C|Lh4L*x(1H~0hFPw1TyDSm1;x?QFb{Pe zvuE7yP}Y=fhltGi!xtX_08^Y43C{?0kWP!8y(ws!$veUT%K^O+WJP zy#uBQAlea1zX8$h8jZ2?&Mj_`J6p;Z;ctF3 zkrnASgfspCS)ewTJn?zv#Ty{=)nr!Q62FSW2T0iV{^o?9RwiD&Jc$% z#N1YdGcm3&@-oTsr^{#;3&(G}k{A{ZUn_I`YOZ*0Bnpw(NZTimGmI@Q#nObI(v>{j?<=J>2W- z$3||v#J{qrchar~#k0z!#ogd>Z!t~*N6C?)A$=7^W2#Pl)zhwk?UNHzd)%m9Xll;G zl^Hi0P}fba6eC^rM%mFIS0UP2cn@O&lrcj#%hCJr{cfg6(0}s7QAy7{74sKOT~b`K zN)bgGZ8nC-ZOZT2{rr**VDzfkdPuvUC;gc`d#x-ly-$bxDUCVcdR+uh+iV?zIROMr z4JtWj6D(E%(9HEYgVz3pePddm8$=!|%dMB)di1+l>RnyHkC7Gwb=Ey(hRgg16v+$V z?|cqWuTb$lk8QL2t{vDQ*o{(LY+kgf1`=B_f>!jZ$9c7MiC@0pWO&}@NXu(J=Zei7 zTsldsrTH+$6C=ISL1PsD4Uu$=h$y2tBD;Fks*SbNL8n)xp^AN~+YU~)uMSo`E!yMF zo5C%0i^5) z5ZaoIq{1bkNTrCF>NhY#o zE^@a!q3BO%cP~QWbo566$A$D(%oA?(l2z2~6t3h0paC_5(o}VaiG7Ps{|itpW!d4+ znxAa`L*UmR?fzXht0KsV5W{9u{7OT_p>Dv!^*S6)bq+3h4PB2vOfGy8BGM1QXQnE| zK03lqtj8K@y3MNh*hyK8X{)Iw(;)EBy19-Pk1SMB7wKoTl8daIE?b5<6yx>KlIvI0!_JIk&+|=t1 z7PI-8j~#TUF0Xv*$!a8>r0R!=5@>w!D2m1NJlZeAv?RqW*z3fA^;HuP!GtF%`|1IH z5B)))tblill$_|y#{MYs`)+h(K96cQwT4$Yi(zH|FxtnY(uE_bf<^b{*F+>Aapj8f z*vmAhb$Iu*-Pe*IeUyuc59)63D+QQre^}E3L_uxrg$5k+aCgc&@T`TG3ezcs#v|%B zzp6%`34Ic#f|PN~JcO=jL!={;Bp4T*I9l%hAHnJ7vq8!Ah1W!a>d~Ld4tzoPj>~`d z16XiXadxeX-NBinp1z9MG+x0a7NAuj?ZcSLzFG5VOR41uER%ZTX8|ESnPsp^!ryc{ z>GzmeYkAc7TsqH7%TDxL`EpN=JFiXwq=Zo`?dwv}7qye^F+edU@<%gO#T&(k`;>dd z@M^)<+AJ~aVPEls>dsKHeZ#(=7GBid%vELkW;&ZRNw(zqeL}OuoH&crDU<>z!CF0m|5RNnL*4Z~CdMfXz{5d4w67jN zcR8HO<(049k==JTNK?rNO-^CdakQC<|_ z+JA*fWO45)K#8WVP>uvJ)^-7KF^(6|qi4Xm2$&9f^oZ!ay}RkLi0e+D?($$(^2}u| zjB5u`L|mIR5+}ljS!$6Og6?Sq2n)|)xpWHSL~J>hUYo_}_EuLzIg^L1t_Q`aOt~?y z3ImU^7DKt~S#SL-2sntS&KCqI8ack z2;Sb3(lEf)cX;XyJfW%LVP(Sg3G01b36n?0r51p|kAPJ>F64~#G6~cCP{q$Hs0pbU0;BhfB1+ln2MWV0Sr=oCshM$r1)(_ zYpnxka~0YtXqsi_=J|_qJgTJd<}M_EQ4+hjDtXh0M9d|uOI4|IrZ)57uwQbI2@MKI zO959cL68cV5t)WAG>T=xJp>ip@?lEngm_^jx)XVN(g?2d6U%Mup7*hG8m=sri$*|Q z0H603rEWsX3MGfvD$C?rbE{{r{`oMtu2!^WGqD(z()kU$1b#?&5vcf%FZcq==|S6* zy3o?0sA9o|XIc+tEUVh3v_aeopF-xQw^m{Q(5MFVLC;4bsG_113C6tK8fx$BB(Qut z;GS%)JOn_Dn$EnS2VJ=DRUt`Oo9JkSM>0B*0^Ud{;QYR%8hlFhhd~!#SMhd4I3ySA z^m1slz2(r&R|Ue7Jx1cgp{oS*+bK}sXj}OM0i^K1{u(!@i9B$%$?=vd=k~{sV$~*U z$D!}f{oRZ*8m{pVZ&y6Eg{$^{KF()r+TXv^?U4H2xBJo*2o#5M zNd??r1@!GrQd_$vx8wY!&$!X^D7~g)4!#{Pt)d`jk$I~wsC04?Hn}~TXpZ$0fDr*f zf5;F1DI%^kU0Tt%F?Y)lj76yb`4usa?)2k)8!meXS)4mT{bW4W$0b-2hAP{oNN zrOU&SB0?WyCb&WxRH1m{!{kev->5$ld5NGoyR7o?)tBas#H`#&qDA0mHwa>T@S>I_ z&E&n8^Y|nqe^l^sH>WrDw#RjUJR<#_4B*8m0T7{01<7O7ZogLpmnT5P=`*Yol(R?R z&T!B6DR>j@p;`0dqmn$H)xs0;)ekdJLrV;-P$~W?N0C|5b=6qUQn07j8EK@A zkb%9l5IK{Ls2^`g+c#0^FDx!i|LqA7UZ4kNuv=wn;=6ImcF_Lh% z;O_xAH_s_H2_RXSO0hj#ITcSbDZW3)lwh{9K}phumFL$n9W-SBWV?lKYMzGupNZYp z`Z~+tEy$td5qf&Q$W59P@Q?+tH|Qk6Nj)l8IOQeUzQZNLDK=@^GFsVc{m6cG4PmSO z9as#EAKQ!JjJpKl38bR>RY^;9P3+w&9)<2Kx$;y}R)64{o^u}P{89KY@>1QZgBpzL z1CUutt3;uzxT+0LT(-slCj0EP6?4KoONaG!q6si_yfjt&NC9B`vbrJ~k_fA#DMk~D zq6TJA1%R7`zxG%(F6l>Qlo|4?OVu|-=5xoMU(vHs660}Jr)py!0+Ck)WQCx5Qo8wi5 zy3kEV{65|=J@%=zf*TRByYo=#r{s|`OF+^1d_p@9YR(-?{AoqQ1~9?qgh(q$rJbrz zv|KJZijo~#ojNnel2@=J)$exlaw{rg+u~1uJ9>fuN^9ZW+CBZXN}?`7cuL<-1c%fuU?~bz-X2mW^S+xCM&Bi@Jj0uXxG_$GxAg(USXOzVZs)E^_Orye z+ns@%GkQOn{9FWq_3myc$7l#Rh>>EW+8K+&Tj&V56+gHVvmI>Bc zM(08e%R$>V_Q?blh5fvy@Y&I#l--frDj znDZ;_jKa?iUda(qUHCk<(XMUl-+_C({446{0_3Ad8{{5t6zlQwkgsMp#hWl@9!znK z{y;<=E5qJdLxUA^uB+;IrxJnoHGNyY7Btb}97d_c~Bs zX)YeE@iY<1+8yBSa{H@Q(=La~Ze1ccc>jTk{Sv`^|6Bp#w}Ewg0@`HOi&_gpca4UW z30#PNc2)+v;MFL$0%uJX?1ngPz>zwQRvKCaL`&wk{aY!O$4k4#fzcrQ@v09_+VzK51GoTW)li;nm+dKJ)BxsWP{w9I6Ygx?z6V_DzNybI@+fU5hdaByHP=rH z$Ebl1p1+Tnd3B+CS@5{IXS(NV>f4a)Q!h`lwqYd;8>hwxzeVorNH`Idw`i9Kba=@* z?{Icm?|re7Jdbno&~vGiJB6DVc)bxD+7R~1V^fRB)t{ZP;tH-_{+J^gesC$< z?7bwZ4H-l^@{gz)0MOp{xEOz4r_&*Gw@qz`)zFu>>)a8yY(@I5UXLo2GV|Q|$CnCs z9D}g2LZt{B%T;BSjlS<|p}652UXIpIT3?5?V3XrUYQtP>U^W-PBtul-t+{3&(O7*a zCd(jGT!IU*-DVu76`MgWAZd{%e=Y(ZGq9av^v@c9$v|{Lt8V$QlpIxZ=J1yx0?`t1 zhurq>QEPeL_Ut=%u{kVgq~x+Jn1F*j~}ZxW?1oxdZ!({;oyN4 zd{`qO)FO6W%&?qDIF2txYI_(vJjC4`mg6Izz1!o$GdDXd(Jh#Os~E&i0+V*zDehj| z6M@BvRZ_q^d5)QrfPqJ_p9KO8j{s0HRO@71s^R=_`zX2nUWd1dwnh6NJ*+L9JiOoJ z;-gzzNR@3#Cq8ljV5-p<(76u&79;X&*qBN18%-2-LTX#xPuKT+B8S!tJ*(`mZArPjQ~fJ-yIj@+{9dH_Fry@~C+2@**vCsm$FL zZ2#!KG0fNcIWigzS|e4PJ^`D(?Quuh6N6gI$GtJvy5m9B?kOmzm@j(H1*Cf4XcJXS zy?iI`Ss#_WsBx)|@F;TSm#$_#Gt(ng|2kHpBFXne%$dH%PQJ#CYo@oMaaX!O(mDIA zU%o%vt$*u-u?xeH*d8BSZT22&Tr{g0{_6QqGg|n2!dGCX;#lXNVCM7MENn%9 zibyr>nD_jPif5-onjG69O0!LaOr>voQV3ZvIi9s!vD*h3s!nY^>%!#WH{mJmcJxQ7Y}tevbcXe2mT2 zINCHDU00j!9eC^-^q5`%$51AXI`6zTa@N)#_WRo&p*}u0xghVD6O&qdtlBr`)whpz zY5$Xvi7EJ@9I~do)*9MulI?FMKG+@c`)mG*Yw2AlV|>U5J$x-*=DG*_=arFrO@>l~ zjbfVXGtMm*U)+ffSd6(cRB(MU$Fn{@fovlT92)}oeqf3I8qNOi2V^YZjAhlL~2*X>5~UYEzbLR}E>`Y;4zAYS)+RFb?T3Z|tyI z>ey>}ieGOHyq-f{>hzTB@(t+viI6jC za^2a#%8xhplq~g}M3ffp^qg$$ZL&N)l69gb(OJjRjxuL9c(*e@SVnO>6%7%QHXdva@B3WL;m} zde&vxEB|E{|2j`cE&%44bsR{T|anu77{yOalFUn0$MLoLfS@aOUb! z`=*NDaW=8DWzMtK8QD#VH-}D!o|npw*{Z9TnT;Q>>;9B-!{3V275u+rjd|~(&Slyu zp}t3>pWRG|xn2F+Pj%-~gjMg-jf;r|S8top9)G_BwY05Stfz)nZ$x$_@h=NB4O|E#?G{7nL@T0ulC%hU(tCiq*3#V@P9uJUJe!?pyaeR16SjN2-^Kmf=k^1e+hwmo3|5T0@ATtTKBtrc8H8GCO7QpM|XI6GK5V5b| zJD#C2>c+Z^@XbwzH3GigROC`mHU0+H=d>lBM?f8GYved zZ~Jq!ILG*ELr1*CGJ`LgX#geH{3;cN+~ft*O)u#?2NWo*62J)!5stDV7g)$`z`nmT z^{;aRAM>&ttf{@BF*UL+RunjS9VPelMgzmoq-I8VthYSN(4wSFY@C6}+a&=R)OrS@ zo*}aQnTUhf%H4~9-shPtW!eo8X%!vFk~Ow17o#-nKd>l{WKaBD@~$n0=Mmf{yf5?* zUpqH=qAl2&(ER+OLW~h>!^6Bm}1Oh)J-5#0n5!UEqDV*VhFIx}vTn zZ`U#N@?-D`PXUt2GP}t0dQ$HnN6?4x(cRC{J%5bb)ltg_(x-p#dwDbe!1EMOo;WQo z6FgfwppyRfSa;Z6ts#OZq&o2R_|cch7-2{|uwH<=LQt=H`h+cWGwbiw{dbP};@?#+ z=`6FvJ@%=@QfjsXBIgfu+RI)HW{E@C=jcY$;>`G0KIo~Ta#F+OD!9j|{oa_&+9!Zm zYtiYLrDLww;w^q31!oI>oA%NL$U`cPVc7; zYT>M8Hnvx-&oMDf-s z&#rt-KBmXq^V#Oe&7#5WL=E8+;h|c&8X#x*KQ=@p9lr_lEJZ@)QJsW-BCQa5<{f|J z?vxTy8oBE*-$W_d`;thz`w4MjGZ`CtBS+M@+pPgKrpZ$<&q?moQu z)5Q@g)Ols-4TZlk+Dq+4Mct^OQoD5iY3(4SBMUCtv;RTjug3dz9kMb|>!~Rq810ug zW3UTd=XbN_v3kHkug5)>lG6cy-Rh?UMJ393$s+$|wPv!=^3sT{< zn`pr63AC#c)3`o#YR}B2=!=oZ@R+`5^IT~X^k~1mvM8KP~&Q|Cyym;yCElos)cY zW-}3w;clABif8WTU+e*j0=6UL&gyDG2s%xq*$0apxQ9I~JzA0ZUqiG~Y@B;RXx0IB z^(K3>ESY()FF;rblP5}Rpste|o=sBDG4mV|MSsgH?pP7TWf ztT}lBES*3FhfnKS><^s_D|%}GrR-aq_kHn`*dkmEKt!qPuUQaGCuctJ5Un?6-G1LL z1R1C}#CAeL&;}5dpnm_m>h=%;UBPUH?*PEoco)Rr&YCJ^yJ_V6$9(vkE(?q@AU9Cr zSH=Oo8@C7RioD&`ni!!e$&G!0OB7|U=Y_8L{-551OQPbpc4TAKd9}g)0PXppXbo9N zG{c}=m>DTTt}(!eXx2Nmd!EENuTT%qFC~;JjuP-uKa%U_;xTDVPFl_fe`(+B8qC>1 z;`5UiYj1A?nEhvDSgCse?dWNo_#pY7(z6A=tgnGIROZ01 z&VaHE_%MidKZcc0IB9UnNQpMC)0eL}nge%YaGCRnU6=Ifhx@fCJZUBpb3|3mJ?mn$ ztieDbf%C3XDk_MaI6f~^I{<<*6lP8ngxk|xdEHnuho>CIz)~RDFELG4H^+gEO2W^; zoj5%9NE%s94X$LOX6B&kT|Yrl$V?prh())SaplPd<_Q4bohNv2U%{sr_Q+---|_xK zkM-rKK*7L0)oi?2YanBa4aR3~1CQ=%>mt6rFmzj>0)nt-2`VRsl80$-f@v#!vM&Yl zu?+5W7k*x86ok(vM2;(3jV|zC2foCNg=<+ZP+ui6z3Hma0@ht(x;^rqZ)8ido+21bo4z8l66)a!?_kOR9d>>B>ZBTM%!Z|^SG zc*l@V{smEEAhJp1(QKz@OSkOgR7L?PA!Sr?;Vv2|+MW=EFA3`A75*Ed8rio|LGuMra zP&;U-%CmIynZe2NA5c=beRpDX1R6s|U| z6G51qp508-(JRf6E zqeOG5q8FH4k`Q8Y-*A!5>XB7Nkl+$gi~JP*8nq2|oI$;6{ zIrqZZrptskp-$p23~l${`mTxiKbn57DltgFMbh^-G*Xt(?@Nt?()A_ ztK7)FzR9}cQ?}+)K4@>3KtNP18(hYOp^1Kfs-l03(4R~LbG(Szd<8SZ!|%NrAyl9a zK!i!aD++uwrPMgyN>~8V0A#4e2ORSTCh<7O(p82&;WDYPTTxPoeh@mvoX!=|B`aHx z>+NSl5tyT8l1`8KxPrT`Se9fo1>j}~O+O;LCtY!H6yiM%s7=TB1lk^8%is!B;wmk% z3`jE#Fi3->X=p-NkS?}ct`96{0dUhFL?fwsY)Bi!*;v+IozIfgRnO78xWlV5u_rkP z;Y1xYKyfa7z0lT7Q{(x7h{>ocg{bOdVAG9}2VXYI6xI7h>|3Ly$Dx!qz_^pUd!}gjbY2M)9P(YF8DL&Tg?vm?fJMrw z7|14KHRAH%QItTy?(MrfhBJ_#TMx|SNrupY;a_7!%yM6U!9`PiCjujp7^pcKFxeR? zWTa=cC1)IvI=Pxmn+68*Ai;Ei8_x&MhV#!vw|})l@E}|?w5!JVB!eMhp6itiVlqU> zJLJI;R@2|4i$);(iFv_v;QDv(J8W3}h7h;SuA+OVV5b0nUurD!(+&~i^D;J-08fvES$)%-l!p5AU*kz0Znr9Q;|H}I<61_u6S*yL(nRjBuRlJqJ?X1kWq-p0(vzvm4Flf9H z%|wzMD7gqj%RmlTl+2bH&Sror1BH}B#Lv>Ev63T~sVkr4?P7?CDJSi#s@-4V-FyIu z0AexOX7Pr5Nz#hj@%#)l-iUNU%1gW-1TtqxK>ZCj6haJT=7=~tgeb$PB654~Mef2gwD+NE$gq0CRgs zxa|@aLc?)Bph{CyY)4aFFqSO{i(6Ui@RYO1sX%n5rn*piuEG;u-tr{GA%iX4a+Th%xENc4Gg%Dxu+Go@ZB zaw;Kp5t0mEM#HyG!W;ldfZo$IpQ|P=_o`F_VJrdh3y}DpB$`AyQ`PTlwFv*r#&+OWch@K!q4yXJ`qR@8mgIbZe6=VsvJ0?*=O}syBACz)8o=^oci%crG&$N*rNVJj2!o)R z(=<76G(dW@yUyg04*g<#nyyZ)B83Kd(xDtZ3XmkL9UxWUdr0EE&H<8V@JuFb%8B@0{s55RXZu5< zbej}9OIUv`e;_$q)z>bDlF{7Q+VS&3pO(UZL>?8Wd|w4$RDHZ~dN0xYfYzm`04cam z6e;UK2C_9e`UWWICIJnCQ&X)cT!VobLXm@qT>e!D-&9vtM~hf@A0AFAv!`VRGT>wi zV11!Fl8IGE!(*~E1YaeB3vV2GZ+PAanmrOEYDkD!SNNnYmop|=H|fwDheKg`)Kv%m zJ*6X)AW_2vZOy&54g15i?ACOo;f3v%guP;SAuSIfdb#cDC9dI5%hS&s?=mS2>IeVL zbN6PFUhS;D%5e|xP5zD0=rRZA~q=}7q8iTu>_Y`VNqaO7sJq&V-fZX6w8hfQ)l z1>i(EU9pXr1}3LycE)1Gji4uZNd?|Gb0_;Jd}WEhI*&loNUvvsF{4NfGi+Hi+z)2X zNIk51Zz!S`5$u zMF(m_E@9=QJqFm(zsXk}oz}g(<9<~fs;HVlW(d6=25$_8uYw8zu)rCBw>8lRCapY8 z0SyEhWy)A3Rkp%l*nd@zl(&BG}=QX%q!=Dh2Gft*W?ZOgi5aCIKYl9i-j*%j22eRw=c6chPNxq5)pW{PfTAeUm)gx&)=TlZXJm4&VILG*!L$+p-G z6X0{EY$NtTK3Kxno(=Zl<*7khFC_ODdkdc4eDg#;a_I|jR>%Yi3F$u%uHWj#N!Ry< zW74e9)j{IVp|jOpsilWl;27s8R{t4(H8R;Sg=SD7>QhFuPI@@2M8q=P2Ty#8#eyON%Ph=~)GT`SmOpgXy9{&?gn)&5_~% zm3UQ>5Y=%~z;ch*$DdNs8jZmq??NkWnew-7n^^sVhvk?M&2XYj@EN5chgs#vjlto2 zemqywa4}5Y#(EBZ4c2&jh8_6xTa3EFh#CLT+aroXwC0lw*qo~tZ806s%G~OX(j!jT zR(>(Fm4^qhNI_wSOY#*RG$mVWX!48XGAgliT+3x3^e|3N(om_a5RhH&xU+I#P}g?v4~OPm&`o@;6<;I7KWAv)g|JHYf}tjazhj zV?p9q=%N*?#8Wnk`2J02L$Ej1V=SHfQlrD6*jRrdEhO@UJXZzIS~bZ7@KJGZn!SIf z@1Yly9>a(2ss(;BIXT7Z{kf7jkje^UK-Pg;O7_K{?rx>LmF^RFf!p=4C=FLF(-hY1 zSd<#N6})pFTg204fasYJ#S;wh>hvDti-|_jg_7c_PN3W>JMfYrR`{pGy@MLBTGmez z?AfH{*5Jmpxtwu$zzsd3+(lQmsu(Q@lq1gL+Lyv9#r6e^!?y;F2&|Ih7a}`!p^tbS z5a?v=EA0VNYXuZWVo)oL=H$5Dy|L)1V${D?kUBSjR@Q!boBX(4>O1M&C^FJ+qRi%k zT>BZxt1m@0Kzj~BL=XY7f&%(vJ)-cG&)K-Kk|nDTHY(_b_RAlaS-JfB$e{xW+;>5H z_sb9M<68gPV+q~I*StJ|8V}@kHFW_I)UGqzTy}v~qv{nedkF^q*UHC+Ek-jjJE_oSkyK1BS>vi|4ixa(^VUL!N6Ku8@yPSWDQWkE<4$zw&^%1# zj+_|p)&ZAGdlerTa*`IPOe~C}qeL3CeOO$2NOLhrK{rpLLd)kA=vuQSFQV`}b_ysl zd&mB&&iD3KDiEeks5Ru4N<;c~AkT~o9-Q(k0;%X)s9OBBg{YR7WE^@28}HZtrgH4C zTvcbKuD|OGW0W5w|9Ak`RLy#4lh^h=U9DAANP9G-Gi2j|mm93p{v{$o4zSK{eSi!u zR=W{?EN}Kxdqv*tX!)@JmGbfGSJndoLkDOB`YBTjb1Gw?6ShZ|h!rRIuS^mVyrI!E z#gUz{dod0wTAGR#{uBuM$M~aV3`FZWxslw*v@RFt6E$?@+UzvdQ2fZV_@gRZfbJgW zA4CK5&Ru*riHkLu_L(K3R3eMlDz%zdfbmr>F{VE4Pxl3#pG&7e_S;*=ZBd|$DbhObU5UVwiK=3smQRcb8Vy*2-R|djx6pNgtsqUe9})(o9ACQ< z@XFMe8vDg9EnfEcz+@}_#;ezUOd&%Q46kL+IXmd83;3dT?b7#qk?1$WE`?FEKuba- zymGkWjUTh)V*tNsIFQ0oqkRE>N3OzfT9cGXMh$Wl zkv16r02P#1Nod&6E4Ol?eGjTDbQ8(gPyv-~+2z?h@Km+p0-d{kU}a;P!KOvS)UyXF zd#pPlcu8^JHG}rvC2QqM;Pr=gs{paUU?8ri7o)R!&jx>;k1Yih(1+M?TF`X4 z;ES5fCPhT+=?h_`oYo*+z=oAe=E>JUPGQeeM0!WUJ;D#v)chQ`L(#-t1YYNk53#C0 z+Dn5i`%lg_cdHKbXNs``&?=1$C1;kNU$j3%>SIA;HqJP0=aAJ$)QG|x(U{+z~+fTfA z^FJa@h}F}QQG8SqE0>HN44&R19=MVUr|fZYbfUgbesHN*1=0DX|b^JK0-5~F54H0(pHlC4WP)~l(Hr2iy|Fxpa7gs7G zo)PN_-Y5j`CR*dXDY^1!elKseJQe}3JmB9XwUhHiE2O~=K)Tcw<(!apZYvOJFe>Mw zJ}5JFkO3f3*zF1F*Qu~>iU#06R~)#Y)eq-IWSNK7;-pmT$nFweb;qV#&a11l z4NXx6Tsx<9ril4A%QA2(IKI0KhU@>K%pS5AEjNME9%Lc@x8Uzct2p<+^G;~D*y^Ff zg$4B1RhTS=L+}gS?HrG|Vik)Fdgv^3mDUFC)610`= z0eUKBuaqGCcH02~(&5F3dhs6k?n2+tFEiEAd=B_=A?)D$%9%9@?;n~it>oC8ygqH_ zQ>Sarvr(8FRU{E^-`p0wp*mdY_q{>3TC!SBoO0jTW7-}^PCJirp+3!ZW_ zs7KpX{F}Bu&njm&9I#+M=A6y_z_rij*X^HFP`8D(w}$sNE&+}DlqAk6-;(b%s+H_# zZ_=cP?ehe%uNRee9_@`U%`y(rb1?C+`)kMz%$~Xu}GpCt>ng5CEoK*fx#&vK`R<^Pv1sp40?*A^ymNI}y zY95)#(W*CY+2oPQd(`B$Xgx}+!Ij&hD86L5Jrn%3v3w+VPqIs+S_kiv)LzHWZL_Ut zRjFI{T#jOI6%4I+XXiRy(#nrm+%D9yx^o`~@35lbFWud_R=%~{=L|8JvX8{bql>__ z#S%QB`XQW!_p%zKfXPE*lQ+Q&O`wmT%Y&-Ay6weu@p7)uFbPdJjI^36`pNVu*@a9*`R(rgMD{f|5&SB8(YDFfRgUM`YhqtMMVc z-5(Jl8r{VQ%Pcm((YBO+dS>*-IkPv4JF9u2DrbIg_rD}vc|Hu%SOl+Y12fNA(RXyx zu5%)rE+aSbM)eJW9T+4w?|iW2(oj*JZU;BM`u1k;4d4GnUtzQ;c{+p!{m@cHuYS4! zyoXs#mUT3b&2wnAtP^4k$rgyJbJts5In%Q$E1jC{l5dR0idN>yOthj%rB}kT31tlS zOp1a}a&MQ4=+~uk*i2^+QL|v+PI;?KN9oOP6UKd$RU=1i*loiy<_3_g(03;3*nT{zf;RS7?@uMU-vd zzmn%Xdgb14@B64A;AoZ+%}sBA@`>L}Ej{DO8Q0oru-(<>V(Dq4tGYn#qH5=x5>M19 zw@Ry@KgERQVpbGRSjoHges!~>itab`sCPTxO+u06T@Oxk7u7tJyeD0Y+~rK%J8r?h zyPVOs(Po8=G&Y}Si=U~K$+KrVFr3>#`ehuA4 zaqm|yR#5M|Lj~7Jco&ZctoIfY11U|@Kg%Us*^6ooq7e54?;(+@BAEu{V4p$9uao_T zjqM7eUxR?j4+s-ckN-})CoDT2xdHdXGz~I<>uRFwSLlOGOoPjL2UVW5Lz5s9pf0I$ z*SfWi`|$M_U*~sylWJ+rJBJW&ODEg=MERUbWC)P_s?qVN z&wsZbE(>eUs2YOncgok7hpuyrI<1>Tt&5)_W%TafZ;ZWv>g$|mElOJbj2+R&O|-}( zYmS&$bfa(T@~wNLhfn~2S8CowSc1p;w7b2tg7MKx+CPJdO=8*P%q%%`+P(6f zVwiE8-TQkLUaq(BMOsYdCaVQgi~H?SZkMdFt6N@|-+0aI%1I48O3@2E)8t|8x*&;h z;ZO8cjyw)DZAvlfASa&yzVo?4>^QUK+) z8R#p#IXC1azKOh`MnhOTb%ltF%8V&S`TN@kOW7AmiHlc$*<9Ftz97Z}Q{H?3jV*ei zAf4;5c-dN$rMNJfGvojDapL-P>v{JMg{41U^VxsrrA>X}k{5oFClegxkEaFK?|ZWE zugnef#OzbEh@#oSP2wLiLWQYpg?0H?S)K)TiH6QnZ$5qjc~Ch5(q$GO=YlAH)yAro z?cclt%O0OPpjXK+25&j0K3qs2yke*2=jFe0>d#VjS?H)jd-Zw$x7sUR4#6pUi~UcB ziqnF;)v?mi^&5+D8hf!wZy7PakTzfWXLPZ&-sH;l2O>qFFUe@4(R`>)SW5hZ)>kW% z!U>79R?a>eFjw?i0;OG9JWw-3WkIxN3bm(B^>f9>y+XQs8;;)`7mS5DKO zf@*PLVyZNZxl|mgI65#^Z*s<}RW0UAeCLwb0|;80UPFv9i=fGBAzqy zuUl=mR-Ii`N+I+x8{28pSM>tcf9`KQH@_=7(PfTEJbJd(;Y&<>*zpc7^YD=pLe(sK z`rz%7DCrsNZ{gTIXa3~XpBub$CGp9UL)gB&`Hpwtzw6ceiQ$!b6CCLPwYy77k_(~Y z=KJ0O612rv%H|G8hpa2UsJV_k-Blz>&b{3LGH2Q+t1VVCm#@za&CGWmUt}LY0@e`r z`&lH0)00=gkJje%qZ+aWN$bw$;n>CZ0w^u0u;rC7?FRS;opAR>X~jz#f3aX}Y~Qiq zkE09i6Yh6zU5cB;imKE*<*)5r@qIV?3Uyb*QiDM`6#~{^;59tnI;>>tuIzo(z32U_ zhteAkhJk+NA<8Z_^gYY5bgYBojKp^f)vFt95>U4SZZ66CW_@ASI{NCdje3I(&F&4}pYY%7QN+LN z|NU7x-51?%v)=G~T91}*kQbfPUz2~Xc0qNv$k^|rKAy1owiS%%3u$R4k=|hsM>+&&^j#l@FX&OrxY8_TJwgbvAoKe%Ms+Vz_jLQr?`u$0;?cQ1_vT%z7ZL8?tW8qC z<;yU>>1R^+yZrj$3t#e6Tk95n>LcLvw=2y@3diajIncySb z8QV<0z^nIrAKmX`ogQNHGsgbsF880Yp+DmiGM*7+!M+)x*o-x5eWKpa@XIeP_8#zm zbwKcW>H6x@j_RMDlg2yt{@m~WD?RkrfzX$lWkjNVW=3_U@6n6_-KA38J}bS<@W~9X z%pYXyUmL&vvUC5odoW|;FJW5Qf#{*{*_q1)1+e-K_anIz>vvb)uBU6t2T}m!nZacq{{_FM1m#58!FOF{g{aWYg8pxEW z`}zHs_e5-vzVFnh<4^VVzG)Wh{X6_6ximAy+$0lz@Rn@BykOHS-pZ`ivd;N$33@*O zt1Fc+7Yf)Z*P*{7E+5p|in*ruPWjh|>ZtFvjeAQH9$tKwcJa%;+OL^T!D=V>`d|O? zXjhKw`1w-!m})7MLymiLz<8c>eaOTnxysS>Gy>n?q1nShz;Wa`-Vz4e%M$;=1``J!=Uy=a)ayKF_W-< z=Lr8@qg7|N<|e-&Po$HGYJvJM@ACJ)x*<9PL zQuAq{tk_=TJ9BTb&f%>SxU8#1U1vPh5AY)m8Y3PfRc^MukMN!k7V-~+m6~*q40!j` zw8b0Vz^I7!f)gCS^2{>{zi(HaNJnE@qdA`kLdJBf#{|mz;}5*!yX@-yafY6TcpORF zVngZI4UBlN0^Q~pGXSAlTp55Q);9Bj-rDiawZKD$hRaE4O1qbi5gq8jr*`YBzcFqa_A9KY9FP}vPmQ5!j%{87cNUQFLYeJXZ~{AmCuDwT_<(&$B;meMFpj( zI~o&hXnv=s$=#ZB@6wk$)itT*Rbq+9OmJ2?z^%XB02;G%0_awMn-12cPpB({9a?RV zT`L_}uIJquA2x7DjS1)IIt<_t{>`zQ2_osJ-I)p#T6zT~tK^*5`a+1{(^iXmAY41O zM-0=Bttb0QvJY>IyxloY8Pv^Zab9J?K+Z8T#;EA^W@*`vlLSJd0 zI*E6!2WJ_18H6r5abKtrAJpKDST*fDhBl>e%!NYP=Su|V#bE`41(Of*meJq}CK@_B^ z9q0eX_EptcSkugi`hMm4$(l6yBRwdjNokuFO#SJ)?eu}Me1o>8$2?$ksqmEHnJyg1 zpc7)V1qxD%Mri94Z<){UdUN*f>Ywkg@5-0;%r~c$KYZ+jW4j{oYO~e*mXB z#wJGpF7!=(>+)47BQ@skK=0Q_6p^L7+iN5b=ix}V{^|f}cron>e-l*=n)sM^o(-k)E(Z4z~8hj;4=|joqJ#s&h z&XF`2G=PU}A@hPHt~jcp3&Ft;p~$3`xOf%rM*53%OFH437X1rD?Hx2H3-B#=O~4(61F8UNy}xoZ zF_mhiA_%Mv@4T7|Fy!+L8BDe~JxPh0IFXig&YtGjW@;&eEQDOI7RP?w0sl`BNCL4x zj_Lz7HlRG81Z+~olT@~lDLr@r+cir9Q3|7*$4eda=9ja zSt2ABM77VlU&?if>SZ~a0z*Jk&w$!t7xY2dQE-xjSSn80dByC$6LPLGpFXFj#2YZ( zN3yEEA|>kVPy>_PaP{L9Ac>VY#>>8-+&`vnSHcf=O$a;Vkw#bb_o2!uyEV8L@AZIE ziSS;gp%mO!aQ`S zsi(PHKW+WhhMy!9k!!&e^QdRU>Zft1x0%VpcPXLWicjje0wy=kM;IS-RtD(u(((&! z=s?==fN{xgefnRTDr*?I*5y#JKQywdjo|@6>zAx+wTvtsJ&yZW@KR-t71eVNkkoMA zyk0TZ-cFY6G1kNEC~5||<-~Jk{9-c*r9Cvakiyg02Hk)1YEHdJVlw{0%7DV=)HsB~ zPT58O9-6&^MyiQ)tV)p~f_Hkj0MFoxw-^e}mMRfoww-bWp(9m)|8^h};ldW#K*5YJ z2@!Q%4g?=VaD2{2@<}G>LK@!uIir>ud=C7^&Bl?c3h&V0kMlfzTBlO(_8-u6e z*F8Ev=PjWa^glQ4EJRD}heqFMsT1ay(<`4?b z0wp7EPo8xu5au~2K|}>mEceeSM3uzAzj+fBf})S{TI0;L&4CymGnItnAQ@6>`h4Do zZ?9;(u6t;P;Mr>%{x+ltKJ*Yl1Pzw6{Z+3FyP7O+Z-~DC=7qe+CGSUR$>tO-C-4-H zTywif8w!P=0vK4dj1?XoV}~ZP2d-W7*b1!|D78BPHEJwN+=7@_?n-k3PQq{NnWpE3 z!VD|4{}4(LK}fgj7abliTvXyDMX1kOT5)nO2CQ{Yarp=Js9WC`+IRQoSlXt3daYCu z{XVTj)5S+t7?0$>PaQ_;@jBrTVE`qY)hb0@o56@yE_%-_Bx-x8$>6d= zSWknoXG;G|?=)yL(vWme_9f+R-sOmrnDo~%Y!Xwy*!$23{)*p#&YNPr|AhB%2kFhC zH6$;0{LSsTD;qG{>CJH<|B=5aeI9!V4J?ilaxjzm4mKNd-2!Ufls?OcVv-d11=@k& zWx{T2a_#ADbEH4*!U_tDj3Td)W`sB{z{d}$%9nhcC%9QXB=V0kfl_9W>#+TY{3-i` zsHp7*xh>3N@tGhcnX-Z%ZN$>wrx1lIOj8&!yT6VZd8cTYG@!B9OU-1lRr~kX_dr z?W?W}GOqyu#)1^qmbOeq8YNuSzY~m-2N%;~Omg0saL>gy0I#xu0~@#`B_BWu#bi#R zI|f+eBU<)|+YnqEXUWzGu@UcVtQlKrn7fWFM)rs;Te<7A(U8Mhk;IaszkT>H8iYD4 z4WkX07Mq)~_2+24zd`-cc>r~Zbp$o*wAflil~$rhkdj9=<*(G&u1r#K=91MV6a+vc zE1>_h7n}vut{2Z6A@-JV$ z6jREck&yfK_qGz}j1fR3qQo)`^eil`T2R}B0v=_v= z9mVFuAho}_vQm&HMopK+F`EOaEWAz?o->=~^x8#oo%!Z`0|hemWxWx&4cXh$ovkZ|&T|MQevUT%_s1(LtJV69>tcl4>X*Iz=mS7F<8AYH{d4>O{3Q`aZIsZ(TWC9@gIG z;O4_vJ7e2P4{A0?0LP5f`fX;TgW4e*ia-pi)uDP|8ppZ@q<{b@^nfh>-ut7@wRV2> zrw3>=O%opXFj}RZUY0PSuIpze20xs=oG}TEAO78p-KK2S=g;J;BZD`jNnA99Bm(&oAZkI=`qbf6k^hUrd(%N483@o7Gd7@yVTV@zwf41O7GOKc%m8n{M$GBE8J`+^=VM3HbVL;3+UQOwk_dj@}Lk1NY&ow?R? zgzdd1#=4N7w=Zf9eA=TpX}NIbn&xQ=!}xhfAf^VS#~<4=9IZM6-eCs^%BLnP{LgWLo8?(t!DW$MHh7uT$7FE>qB@zYdi?x zr&+d{V~Ph=(I}``n(+fR%$ZX4n*vN>!+rp3R4M1jq9NMbwLXg9qUnJ>Z60~9v;~pk zb1O8F18jX^x`t*X(uPuETrf$P|74h1Rw5aoK1Vy;Pqkul?!7q$0!XZk9Oll^H~AeyUPw)atlpGh~%c{#M1~#uowNmRY@)# zq5-(@p-sZnq^GoYJap)UpKiYShWh$AJfXblvMMC=gHdt0p#lUzAb;C@nrc;f=FZ8M z3K<-1wNzYY6gR`f%jJ#PJO;bX2P0BDH;=54=WL=Bbl@n%59 zuiYU}-=M&iKu45loxY(4ML?5o8~s=gg|bP*G`KSjA_igr$^X^(?Gg09joCAW0NT`f z%^3|7erWSN1mrPoP4<|5-g=J-8hb8;+t-D$4hR9YCQjqQ4J;p;z(#4EC>X-vx#*$bn|L%x)gdQ$)$A(QIHC z(7K27ejZ26qW#g%19nl&!`iTgG%bM86!KL&aWAlyje?k2^Tgl>B6Bf;XY_koNBQ?#MPB@x*X;L( z%e1fnV7D!O`dX*&;G)!@J@0+Oh2F0twQ|eWIKj~B1DA0GzQFDDQ`{0$8nz?v=q^LB$BPe5^E_?`Hv(#e~9&)YsA>e*-hGg~0qh^;B;XCr*;zpt*HG*&PLsQ#5G?Rl)cmFv5c^Ke3YJVa#v zCka=CR88Too8!tw|7hA4{ANucxCa0>rKTQtJ#eGiyhr@ns2`sWvKaQa!hsKYUIF!p zOI(;fXtnIpLT^B^g`bGfqo!D*ruE?QfwoMV_QK{#gc9@A?`#G)WgP|?y8kOx<2^_X z0noYC3~2f$EChskBWOYK!=9=Hb#Q+eKxpW40FKtQEEW+aMKZ^m* zA5EbFnXx_e(lQQROf7^G$V(}` z12pF#C~QK%UvjD?sk{iZW7~H1^gsyCOiSC$stIIOSPMKPg0x<%xwc`u^hrqO0idj5 zdF}g0G1yffscVDh>EWW44lb6?meB`QWnA5^a9E1xob~}YPJ|nq5e(!3YXEUz5v&CW z9HPm70?Bqj=w8v*CSc8p?%l~E7>fc7dUXHhMl!F~M19KKLAD$#5*#kW+IfWriMEam zMAM1H654as!Fvy6`}QB6Jb2j8GE(y9%qmc_W7Xw_zc_LgjMC`KjO?8>@&M+vE5$nK z_9$sF(JGGaS;huu-EuPOfx2O6?P;L)9B{}2m!J`H+V|xp`A!>i%6}+u1)ZnLvr1g~ z_2+OgApP--Vt~?DhS8u+X^MIhD(|+&W1hVN>8z#Mg27Ljbr@%W zrYlAlf=u_doe4S7vsmJB>zmG)efTZ8#1pB_{sC7lr{(>$V ztbui;wr&kNuD>e*L}s#W*bPyeoHKBny-vPXK`qa%ctt<4rpDdtah!eU1%+lENgL^4 z=b49Es0T3x5Y;+>+}tqqQpKgm72W0ik1GJVvx*pUtnx{aqy!PoB6!KLe#_+beu>+g zL6lsil6Fx5kOP6CG!Trj)EEg5k`wbypMykr5VsIolPMgSZPugm;6pXU+%iov@x})h zN`Tz?0wFSlDtb0U!J3I|5SX#-UIsejVT1wH6~-M0y(=_ABOD3^Y&#aRk*!TPqk|P4 z0L0R;ojg%_*vj*mVq=NNu40Y$`$RAr<=I>$M|zkt%sNGO4B*-%Ol@61o|1arZYt%; zsQpYy;6uj`9Zw!^`t~63(dOR^PabUrV!9r!-#5eMIK(#4fOL+;pt{FGnZmyhV$#%I z@E$%ux-NkN4`f7hxGGzey*LB-oZXcHY3D9Ht=tj-C~$Y1>VFo%+Q7?yzp8B6T&(_l zwtCwaGUT`|W903zKMcz<5y-5!FIi*SK!qluN@p<_8FTR5aYI)Jfm8^wMo# zG?%m2JD{+r(JDj=dF<0CR->;jCQr!A=mz%!T#nytZ95NfxXx8!ua{K2v99bvO+wB| zQ9WE`ul2_;qeTe&LL710);3OB64i1$5&twx1yCGidSjl%Hzjr{i6j?4yj;(p+GUgJ z%`HJ^{EK12!}(6%8~gAaS36C>O|J1ftHt2fyA<1gHNommuclKj>pw)EZ~xd6y#+#P z9CO|8Us4uz=gRQYPL`&ts=g_X3S57r$N&I;Wee(956zlW(}v)dtZyjZ1kWqldVJZ@ zk3Vo}f}6+$^tkgl>1HD1({EztOo@9YGofug+hgv;a9W#Fa{JVuY}h3xgt}3Dfab9r z7eHpG@0Y!u#~&q@2a)jiBIOHRryfBRUr6&X0Itli0z(vX;LfISn22)pO$8WHBk=lN zghj4$fM9?J#qrpqFWxBg0N+>$*e-aS52DvJms9Cj|kpY z<5Thjz&=Vu0suG4Q4X@RYrhPr+_WFIvdd6c0;gfgK!E<*_I|h-Slg6mQYYeAWqUHC zW67Mjit&LvKv*Jnf3&%~F+ppp3MO6|wzhQD6hxheSYy z9Z4faE<-@^6x7p0NCWSYygM7800Ubk3^adMM|IYyvWX6q&-W8Anw*GAIad?%8)^-f zf&z!%loOR`QL5iBvx{6${oNO2;Ks5^zL*bW5Obqm#T6bKO*KX1K%S|#?cP0aAZmdF zmJb&%uH(-b)Quoi_LVDDh#s5et9!jpncW*%qz8 z23gefFzmbyh?rBZ2t<+AlEVR^6iu6--R4DN-%q2Sv`D1ivYWkB(letMk?7;k{3a|e ziZhyMDh8=v>8?B=!sLs@5&O;!pRHdPiNbNMEy^o<|PnS?RgTn4 z9QW$_5#v)O0nsBQjJ+Tg?X>z`I$vvsHvjxq_Gn^cpN6?&MIV^7ZXYsS#(~naAR-7` z<&8!OGNU&yq1;>{n8krIJVG>9B>}eeR5hU#q$Uzf1>6o}9zX$93&#sf6yuVj%7;yP zM2Jf(h^hOL%_xdSkeI-=$s^cg9|x`~l#XQ0C-iQl!tA9$ESG5rEo9}J)dN-K4q(+c z6dvDBqFyScnOMzetw~TY*U8Xo$IU3s*mkqKWUW0#;jrC0=NnW;hBacxk)$yxx}7}G zVY0C&RC4e19d)SGhk0y=$D*F#WC`veSwSAAeghoH0;_$I7JHL9)>rRvZ7lTJqa{X$ zscW$)PsDYYeiO|AnkqT>kD``iRU%Bgt-tVFE;Ten;{pMiimE(~b46TlGIztjG39Rkt6J@#ppG+RlYFTyL09Kj@R`V6 z9`E#;KK0*RQ>c0Vr5#Y^oCJk1QBnO3APXtoDv5OGd?{BUwxw=4%K2FflZW@KxI*=n1j1)){|4%npHN8h35okkcLI(ZU#t>g&Z z!LhfV0ZN6Q8-^U!ZoGNaPD;&l@YM?FEYuJrdhQ^LwclJlMrKLb%POu~5^0l8i7BvF z#)E95!!2Fe;2SS#kQ?|Pl|Am*nOdq!2D4cPx{F(1L)8rMFGTP%(H^!Gg%D=5B0~V( zXd%j9i({Nqz*{`9hnTP*0I_-^=mk*~MTN!-up#2rL{pUrI*-#($8YWeZCsqrZK{MOH8>#9{s_LN`)M1Xn&fPz6wfd-2 z0E%Muns*hpAncnwdk6(^WB^p{BAX102(dG&kZ*d2 zs8LP=r9&LV5KH;?TZl}mpv6JBi3xr!U>QaANHk7M-2As09V^CEf{aw(tNvuJx8g!i z7+I}0po8?4;>5}|fL)KhR0U+mBLkGepqPnjEVb4LZxX2)?Iz&n1 z5D@*(4=5Bx`xp`Q->D3ZMKjz?TSV*59Ir7K;qaZBI-iYWQxMf9<0ScP`A&dRDplhf zKrvN}vCRV|j;kG+QHvFiGmUbpl$4~McIq_TML(Pl3~ow6oEz60@>9P@2Zn(%a(s?g z4TPGzJj*eP!!xnWhPoqpALcLCJe$*Vld2HD=_!A)p1x=RXgD5VVAT_E;`PqZEgCON zLk8<-t-Qsh^BayU!f}>rN5uMg6UQ(LYngBU&KLmDS1Ta`=}4|?Crp-y6T+|0(R zF}-FbGpN4N#s<5bhe#k6X|J#Rn}zLoZ*Yki4tR{Y_Xf(H-60j>k4R%G`_vM!CLb?j z(8Je{6#ueqz|!?qSi2*hOGf;*s4BQldbqu z;V+81tZwM&#uS(Wz;yy#OtkHKebgWpJV@C@*Y{ws>0}Bda1LTJ=aoTJdBFrJrl6Gp ztB=_?JOmiS^#tb`J#_?FlL-aHZd4HWJ{GYzN|9Buga9hOn+;ysrC{M_o2^d>01}1) zApzta4@HE45pN4KqrK#T|NRtTv zv_r0GWYR8@nSoWZnVX2~My~ z`4~9lgQ&jDWUiG*c@6`9k#NU`O>@e}%6?DSQ zRUgq7%VGzeYmb!;M*hU_ITUARxO>!x?7jb8=pm($hY;4Q${8ir_%H&%F&GybkvBF z2Qz&5&h>!x{Kof&c8XOScWL=DZSzTS=}9OC5VN|^k#(X55BS%yK65nycqLb|k?m9j zxz$w+4lo35WP@9uIBD=859FVYJXrGb>40W@-QuQl%R>OalT5o42RUJvXG}!Yu^|rZ zTvJhGFfp1_NYNBQT4F&=PYX#g(*)qk9llj_ncDR!@=v1^s{k^+7q(FVU^UUlg2C}3 zI0X&zXxh4+65AJilVP|edcjySU%n9N{da#tc6Y+zp9x!o>{PpaHmx!3+k>5u6%3GH zsiMJ)gbFLOfcu3dKj(JLnRvgWC6+0DQjuq}5Xi}y3}>;`ftJjT}TuydDN zd6(zyZo`v7yWR1Q4syGIt06A-BQ`ZA-q%9hcG_cBrqL-p$)5Q3-# zCN(4B*BPgCJK3{tCf6CY1_ydWQUhAZw)ReQ#_O66!K5l=UnT16M_bBdE#s$WpqHeJnq4Z^qbDtJx_Jh zSNfbc4bw02JPPUY?oS{)&z1*}qjBAwJlf&^ zs*fFcyZygo3c6xU>u2a@e4_U<&)*^MD0-J_rRVLcydBWAQz1vXGWKq}=C~1Gy3kz_ z;g^ZNp8xwNdWZZe3$yfTVRA#m!`g_Mca=6)$HG|G2OXa?65n?p`0y+#>TPD)a?<9u zpt3KaBtqde_5C~4iq9?$JT`RyysVy zHr9lkoZX?$45{hpejLW^39YTAppO?eQ{(d<-~uL`m)N_~hYh>yGpeb3X4n7;0a!5QJ0yYW!pQ zRPb{BxZ~-k*{65O$M&eGWgU3tQtgw^Q7e>JQ6Di?qN%oi_m7X)lud;fe5mz);3zn- z_R%_^Eqk5!j9pX4e%9iekU{V*5O!gMWBnlhp~&xZSkeK~Ve|pJ4*=Z9Sx9vf`)i+< zTSablX3}rq&WHF4d>rsnG-ReNg>eFNqqTIUwcM%AXmZa{ zht#{j2kY&ne;OI{N0Rm>vj~6)|C9rx(aN*;5c=~|DdnTtG%c; z$!1gMF6&E0z2~jBo2v7Uy;3zP3u!)TAE|q@al?k5n~B0H!|VHNPGpBU8cK`SK(jg< zIu$l;c@NoRx~Y4|!JEA|Z{9xWL*JAZu2m5HU&rR4^K(rVcQ^Uo+j0I;&h42$C9O9T zORHNpYhG75b$MxT8)R?~XH!|h!IMSXh4Sr{dIx)vH+w5wK!(yiOV+#F{_EcT^G0wj zZ+1=Jxjp&0m-`;z$-_v;@*-3J8lXyo&&F`g8$9njBPS=YM zipp+^6n=?b@2Gq`)T?!A`mPB?5?=3V5;0KjJaV%BW}|}-#s1>xspPAt1l6~E1k<Md!vH+_f!9~LeI&;uNCQEcl3wv=OUXs)cy+aoH+oc|@;ik5 z-QrfFic{rBe*4qQ%)Vnf+qyfZt|mTd&zs^<;hKTA+#fG&%qL4TH^=|#q&U>MHU~$kO%;^yhaL_+ zb#!pgducAt&2)x`5ApENiqM~*roLR7}_RG$nH3A3&9LYYe2vM zEi15I8WREP{Z^JDU#r&a9{J}2)<{7N(V>}<<=WaTp^YDG$6(R>G-V~u(AKO44qP{E zB0pH$SLenMq3-s^dEHDQlSIg^Qi3;F6fQr)7rcpXiR68WHf<sRr6U)n4eVUgbHJfAQ72r_8x=IK%Oa<}?V1Ql{(SBE_F>wZ z$U)!bsN}ct7&27oPdgP$$-%MlcG4Ch=$pWV`u37J4`)|T3`Q?KrsSe8Z;lOxJ1nHZ z6de`>)`Ejlp_vjLk9V&zxmBmJI}j1z*6+~6DF}&^a{7>SbqV~G-j!2RfoP#{=z`7y zKRy`GDjC8jy5$Xyx=7R9`m=`rRt}y%(H1ib`fM|xNEfaJ?DG1xS|<%(_4#R7!o{U$ z5x<&5MF-z(yi5rm!fn)o$Bh9s_8L}eYb@*kdDIJ{dJU#EtyXYw*+1mZ<6ZHhnR6on zGa|U|fWI_&Er0X}?#XB92M-1shF&Gs9zN|4r)dw(-1m;j`+x}e47e80MO9Mcd@(-a z4-QHB@5{R0I^h(vM4c{~%`g8oQ0&59EXn%S_Wu1gysc7v9f|rc{;A1d@ETek6YU2! zLeE|Lu~#bv*e zIOC^ZAz!h?(%iO!3OhFhW__>(|Mwm5vJ$_csK0^j*I7Ixe{=3s9t)_LcPv+Y)Z^!xt*Z0B}}qq`l+c}s58QEeYof!!0HPDtgXiZ=Ar7KL3 zEPJnjWz>?j&cTkWbUH_c%)w-}Rua#9jwWsYUT38rt84oN0Il~Y8jY!tmflI+lH&K~ zoTL0~+Q9euEZQe-H*iTR`M|j= zI%u_ffa%|O^Ghm9^QO{#hps`pO+5-vtJ*XIjOS1QueONyUXwRe%lIayV#P%wM1x;q zmqL8|{Fc#^^^69HQI@es1^Dt&Izz$fUYv_5@v`~ywQVYGqKEkW+`{0&Ry2?~9uaH2 z%5Fby!4mA20@kC(Y%baz#8c~YZs^GS3C?$TzQwV>%u(!1;5CDfG!|CyI($$+)xkyI zbNA#sr(eBec_$QR`}668`wt}W!+*wRfUI=-8Jh%FDj~~w-Fl#J0#1U&)^#k_4kFW5 z3bZ_S`zm!(!h6kA<9vv=%YLHhJW?OkX9;W3`iyvEwhWZ&2XjG!l0pyZG-=Yn_7?Yp z7gJi7`rSFU%pYdz-YTlfk8R?RsTICvhIHD7Rv))t1LWhKCHGI%J#~d{zHs#{?6V2Z zIK{d(G<)0y78JEix6IMUJWtu<(j;7R&$a(D5uU4Fjw%uG-?p_@wRLFJ9S&|vUyySvHrvE$uc z=U<-CP~J`&c0*sQ`y7*W@XvozMSIUHmdOei-KXb$uHTr9RN8m-&r8|y>$fhMqzv#?|^OwbpQus6CYBZ<>BGe59w}#OGVyx?6A5AN35L z()z&P={h<7`-fK{bnyzk%4kU*?b*AZn~+iMY;5_<+C3(9vGmcVEhaa1N2loYS6$D3 zNIGvDlBKdc+t~EJoqYLzE-699GO{3K|Bc57pMNekb$a)>DEl$A`#`bt%STyL_a^VI z##wD@dKNjd6Z%Vxxc54figdwC{GseiZ5e6T_=gYp{ z{^dkhrmftwmwXy~zn^&8%I1<5|v=S%L4f3T(0$_0e*&;xf}L zld9Rm#;ySns^IHVp6)+1l%u;ranvMJx%}w5`2+$5eV30}k;1$(GJW+G=)(PP^^=S} z%q#QNbjXS;0t{#zeWD^`v1yNUXod=6x6gTAdM54@(dB4gbV4YuAcMQZE$)v&9!UZy z6CKplS8xz!rNyg*#*tSf(Qkp+4wobQejPy0kY4`6o|eMO8ufNQ+i%gT=Fd4IA!Da- zdU7)A^cXU$u;4Ut{dYM$E724u6cnca+8?2=>`R5T%djWia~@0}c0I=hvXIp{;u>A$ zlyVk0Yj=x{!Y8u*$T;9iB6p*4arbjTZi_~D&o1v}+zOY|a2{~_jonT|U53Y|hd^HV zGtZ`Tm>2xGUN_E}_XV-WN?)!M%LLfAuvlGJ$gO4d(#W zX>3Sr03Y7bL>*~LIQcyO){``Lg|-pVrKh!6c^Qi{#ob)Su7MlU=Go0kUWSv#Z}wZ= z?9S@V6X~3QGguZ1;m4)={vjBIQfLU`SfK0cG(a6i&goH1#>Xu_Y~?KH6|GaY(?=kmdRL| z3@ek8_Pyi(nh$h#IDqocq*Ka|JjX1J5g}suiU7+TJ8Wl)CiNVXh5$AEK*KVuAqA9` z&A(E(YhDR#%LamLfrdnINTEyV`|2a%TuhoDUZ$`tB+)}lVPb>Riw0rCpoG>0&3@47 zfRZ!sOUmrE%Hg{q9I`Qupd$iV^X)em)gAnWeKF6A%)+ezj{6l79LDOl{*F2|Qa&r; z?D-jL6QH$bbo__2iFPw->aP1u3ecpJZ~j6y@-d_N_Hq_xUaa4lSgaug+57n)HzV0d?J~+S%Vd;YFigfe+5fn4tgU2Ue0F_p z%tgo=IP7UQcoa~vEUewVjC~=*p>Xh>9}ml=s3igR-KJe-SY+#H5;o zd(;Uj0?bjr^E_!w`x8J03sV%UcsUGpq?cwPS3rGz;#fii4+s?x+yi4+xJf?y6Xkpt zQjb8naEc3W)>l|zo`{8VGsfFj{IK7MxFDGW{3+rs6B}HsFl>m+t2}h9(Lr^`$p!FX zv`it{5!=E-E(_e?P}rp3+N}qbeOLG zE4&o|O3hE*73n1iu>H7#SzHXBl9L>+c$$BxVY^2BK4jAy6bQ&o^aOV2Tv!kyo2%DH zgkU$Fue$4pwUXhdM2hZOYY|2U&Vmm=3cyUu;74WHZ0Tkb?$uN$1s(v&C<3OUP?1u| zWg*7yKfrl1Mxh5YCPO6?uh<30S0x?+k0ZA2(~e`W-5flKukez80oIFKl!nCzU}F~@ ze##W4j&;b{49{Qq7h{-X>pP#a8BJ9Rcu#1MFn@6wJ1$ae@xxvbA)5fOo9);s3ObPu zvr~s3%u@JA$q5s6_pLkd(5d}k1GZ%hp_Rwbk*3_iVdQ?7+Q-aNf!MbK<3c~|q|D%g zSu-H!Ler4OiIrRVIP4&iSU83lWn*_eM-&QN@xp`m?n8S4y5%75I9buR1GV|Yx)&_i zl%L{LKbVT#8TY~uTZXfllA70q16QQt&|AnB0KtNFk`>R>CS~k70N)&^xLT;2!c-g^ zgDNazBLSEvY?l_WaG-w`pw^E zmv+u{9b3e&h%VQ05dG5rd^X|)Qya}j$oc39C)BbZ=3tEgmQC0*13Sb-991P;0#pPt zhreIdqx)fBir_RNLPl1sT6DMs;4VoOs)fjIF#Mn37Kn4JX&vsT1+LJq$zV)JZgKIh z3{g!)YzXHSFDT%rbe7P5n?&$G({Z=;Z&)8A{1RXwQt#6DR`Mn6FA6%24X&57usSEt zE{}EeW{yWSqE>|de!!!6CTfMHK&L#0OwX12V0F(J2rfItY9on|HMaw zoXl^!cb2ehCia9MVS#^8TAX?Yh{vJ)_({Jq&fiu}tS#_*m1l z5h`JiAFEL4@k%lfVE=gH7GybFybUARh_oz)6{0hN0la$q=1KWD=KGkczm4Lf%?YVM z_%c92E-We(0lq$HT@fJzBY+M}?DHYO=l3`CBP#0W){ej7T`hpd)|h+l)%#YSSsuOz z`3BeULp3c}z4JpGigpd~wS53S^U>8(A$j9Ea9>BU21{Wn+vVQ0RzIssXYJ$;QFqtu z)WoeU`o)v!Mn(5~-7n9X(r&NC&zO=6Pb|CvTo{0Fz9|1RD zzcCRn-L+1=K~=M=S{fYhe{xv9Salj_SjMUG-8 zv=*-?3;ugU>eWXMJeTgIT>o)I%&U}L#;2TiYIfxlZpzo)S!OBx?Zh>p2=kby_%Zb4 zSjGD9_)A1|K)J@j2;3E=4zti^bVC)Bj~QVhme}C9f437}CWGI^Urx}y?7_#?0Qo}2 zJ&pE;QfcIv^c?^)wM)^64IGf&-#B={Kl<6pu;74XlFIav^vB2*;ppQ-rxWIGkGg4X z->pO-iwQ)-Y%cR=vE5<1gM$Gga?u%z7yD+H!m%V;B&3F0{xES`KA9A zCLd^tBTy!EKYI7hsoKqx7Kd%oPl46|g@mHlk8_goEKH~wuAbPek2uJSzTl@om@-A2 zoOyNKb8b?A%`hdPaqt!9*U^s+O|mYzsC&EoFaFnNEz}t1AE47;iM>{`S;1i25w*6cS`po`#CtSwwXTpOI+YR2lw=Gs4SGB;6 zU1`KNggIo)y#DKyK-WnI`}fy)0@KG9xZ7@=uExC(a06!4{t2278>9ab9nsR;+mmt2 zf)86ytQ++!Be0^J4^=%s|2C`*v*?jrbXV;kvoqQhx3w&J68G{|HAXfG4ER*zKYotu zV=faR8!4zJ8MbLGB`rL=i6CiZz(R1?gk25WV>L#Cf^Q8wtAc0V^t3>aR6K?kkcBvM4 zLw3(u^?Cfi>~#0XO;K8AN3Tz^PksjAWYW$jp=FY-Q1p$bpG4BvcXP2uKJ^~a+BcI{ zaQ>}+90~RCcSzNr0^iqnQF13|=h?BDtp$fxH*n{_owY$(Lw%fU7xjl7a0VF*eqFi$ zQ*`dpO#goz-)1*6voUgS?sLsONwqQeOYXN^=6)AKlxmon`)w{s%w0%`(tSipatk3e zqL4(zS2z3l{r@@de?FgcKJVA_yq=E-C=+Y-kL$ z9Zt=^*W2x>>TL^=K6dP%A8%LTUF`FB^DJ$k5_;?JFZ{$A2w~)w$3N46(pShz`5Wn} zeXa={u*vjsm*jhwRB(~;if-PIuSJE%)VW@|pm;%3Q=4f!Ek1kgj(IgI_EhoQ^3mfNDSUxF-Kwc?R~i(`C(;{KAqlhiuC(2DRW5CX>LUIR+c?Pa ze*FINy zm-hxSPBTV$7=Q5PB({wQ3()`UhVQE8MGR#kqAag`KVI$xlXHb!d+ zHk-{>5x*+c#PT1xfJnc#VX?)$RQTHreWi&>IC@#s&0OD*N9U(4ha}(>YiA-B_tVZy zra0E#Lb>~=y_Hr#R|@g9(w9aex5&?Gs6IfhCvV6-C_xKd!^+|(>~cUk{uZS>Dq4Pq=MKz_;=nMjo<1xouG z_s|Eo<FO3uzEk%uXVAb#M8Mj;NS0b zcBW)!CP+e0N`5q0)8p<~s;$HA+sMU)uhsP;8Zfv7=hasn$2;K%Z%-zluWZj^Zjo9l zs{FFrkh|jV4cS*SiGy&2D0{U^%+-%#rdd^Eb3cPFemnI5{TpUXWxnT=M?6$~X`<}o z%X)pSWCJ4PSilN&qS0%HDCv zd;WJY=aPE*BCl(wB0Yv$tpmxDqBSXu;MYB_u2mebrJPm1ruZ5E&-S~y0|07E;gTgE z{*MV@dD~>_GFG#>u>$pY>F71I=vBB6ircWx>C0LQr`aMO=$xMhzWW>Z&qAQUn~iXM z7)m{dWiszjR`?ONlw&`WxgZ9gPn^7LcKn4D3>XaQNq_H!gA^gJ0{5=zB%l6#$JapqT$@*3xgH(v%&S5P1($E-vg;f>4}lKFysi^LV1GM^awIi{qF#6<|fzCey>!6Usft$EX-X;n?5F z^gXkFsqzBhEZ@(^qm$}@SB8B&L@BlRLhb^84nj>&&h}iMuPH(*6;5JVu$=v`7WC+W zD=5Pg+U$=)XNeda6o@CuYDqC?&2)kJLtKUBR~=k77U#>7X(sr?^?9WOQ}(6Th;`zE z3(KGik(Th!vq9p@t)w)RPO-fglgruNY5+0CeDugf0RU@m@uHVm>Sev$ny_* zEd}VXL1=@H=E-^Fp}0vh)2YJi!)M)M9Z}QjU8cnhlCOG3DsoKrVjZh5e%l;lcfiGK z+YFG*QJ1=mX~sVx@}iw(WHk+BOJa(xp&GX07k<`+eJM+ur2fo{SSwa}o)l#~G*RK2 z^M;y|1wV)C99R4IQS$4mgvM5Z_4NE#IYjU%@u${H$z-2eWpuFFsIYxqm%|NvEsp^b znQAbLk-dn?RIFjDLe@B;GFUgD!~rl1ipFtW&L{#oPc$C3+aS$lh8Du?I;FO)^U)n^ zruy4FR~SnPB2fdU%vXI^D>aVl3|Z^HX4=Y{n;*USiq{H*Dfx;E$3;F6gH&iEmU?QA z>Lt?^w}JX}G}jHAC9p7+#xM|hC>@$fJH6iZmRI4&(WG)iw)@*>st2R3ErD?@p8rB! zpO-tlNT#`;>~p?r@bcvC@d14t5R4lJ>G^kjA^8@EHK}V$Cz0c|ZKUBw@M6eAO_Dje z7Zw)>K<)>_6!dra#OBTtOK~7UcFF^~a9h@wJ!LP%MYo}cQjhPC-7418;PZ8?2YHLS z*vK1}^~BoAaeh7>z($Q_U_H|yni)oFHYpLQaVwCk`A}6f<$Wx_yl4X6NQTW8vYk7x z_0!oHIUmK_mF)}Rpp#EKHou(jZ&Q87d@1GJ4pJ8$ITo=~7AQ#qntOu|Eniv+&d)^| zPyTE8^*zH^vW5%OP7$9GH^h=V_%t6das)7P5cD{o3pyF%+zh+dE%42u@O1Be8v3A>4c_8P)#u_ zJu(<IQe8dUFPrft7GZhBF|TQK^jT)tLogGD{Hjc z{md9T%_u6h3eO|14ow;qNFdz*4XFP?)~I=qnYv${M+T;8K0;B_F0uJ(zY8yQWME~hz}z!=r}A-T zYiaKQ>Gu1~dm5-Hb=UVVzZC_VlI^)oRhmbc!`oeZw9@}F3e?s7uP9v>f*jpW!Ka} zXs*<7HYO$}bxKL~njb(2&gV3kGo{{ti3I{+EMhu^^s&Yi2bB5-%-gjkjFO)ED_zqz z-`FufPD!Y1Ha&N;^x7IUE+rEm2KYRlZh+J?dYf*9MvCh59rg$c8IU&=xOorhN-$Ks z2b|=3n#ygIq)xN{LRHBE1&OsK?xzbLHqZC(NoGLx*D)$<6a%;C+&cxauLX`SWzwXO zGW)$^=GAnEZrN<>*$V5q);A#OoLP77(IWcU^_4@bK(NL?gi{^LIN&jL5AXYKGF1{V9 zk|1Dfbuh-7?v$1Oykc^SaDLJKP_tqd@Fm*()fZ{N4x=hqxazs!9=Zlu270Msustbo zvsdi~g)XQL5}J7fTYC7n1|ZG^g(VrkH8aZ?Y@gNBZ5({|C6@QTX0AZ~<3%>7PB6@%-a4S$YNGLq(#vkGvX0qU|u?knlyPaE=*^p0qGP17_2Co_{hMxPOxp3nW{ zksDn<72$@8hIDNqQMRl43rv+i36hT((w!Of&fsa_zZa;lx7%eIrSyqffx;lG@iUq9 zxc$^wa-Cm)eTUn$_v)L=X--nV9XOrLr?7{PKDX)CIxYwOZ+~~RAI;9iPP~aJOBXKW z-~8QIy+6W;u3d4xwQ<5_QYnY?>p@l2$(N(Z@suI~h5T)!K*=sp=ADzt4cE@`}JdmA#pb+;;BK+msF@DF)fumFF# z<6PBL(XswQUdZHmN1gAE442N1?k9YS+K(5xYZzA$Rc767LmmP{ZjDX0tkvRg+9S5l z9qfLL&L_jg$*=@l$md__mP~t2?%H&9@qA;jIcvgYXjhr7!^rUP=pMQ`l=Du?y(qz{ zy06~7-^ufJ@mO8E`xbaQZ+GVPuKSLJ_jlC~R?$v#Z`@{9A4Uc+%&ivE^^(8)q(?@j zfd>|QjlCUzKlnSbX!<~KyqJ-D>QPwn;=che(o*%e8)h4E*zICe>(u*Mot!=1p0B&P zfLa`bcfQJastqADc(a79GymlYSwNS~b-EgbOYx?Z*j+Z-xAtIVDzzz}6RE`Bu=jpZ zyzprExL7@WInGIH>Hzn>w4Ms#8n2u8T(ey z5BSgz91721`d$xZ^e*@rd-L7%P3ayq5tx*fDJsnzS6I$;T+7NjN zc?@mJ4ZT$v+T0e}@-(z{B=q)TXxmokorBPJR9J^(Sf^Up-P2)RHeuafVfP}#?q3b- z$qjo@8TPO(?9tP($0K1+7Q>!yg*`h6W1+%(?^!2EhCe?Y-XHj(-z)sZ)wP(&O{cKc zK3cf_)$p25j+lIZ%tuuW?Q{;mS}o^<;0 zMz!SX?yA^nEEJcr>J^!P-L(MRp~^q!>bYpW!~GQ-uo90ImSG@SK$=LzlC}S8VB~4R zkDB{i%E8lSCZ5)_8jgkVZ!eTr1tUM21pd28=Vx)r&wTjyf=fzod#j!Sy%vRmu99Bm zXxWzj1!=IFpBVn~8&+d9iP%A+Qkkqz@UIj-d@8u~vtZQrW&?$0{&x0ulyyg^GW$Hj zH0rY2P6z)owl4`|Fe?h;MCx1DPg zWj%*is$QJkVumn1Rj)~6u7s{w22x>x^j@YYlY5^zx&K6=#0jZKj?b|dZNu)lo5yD< zx^CI2i*QcBrP(0;_)LC6FN`0gMtQ99bwoundN&uO*|_bZbHPXvsFU(dQ#eg(aM>nj z>YCXF118l{@0{%_NRO3jhvNbqU8qis=&1wfyxrdZ3p>urpPSxRd=}?%4Xz@b#(iYz z@G%qNw2@XXgF6hrGcohfpIXOxBbKQqGmRO+0DZ<{o*sA?B}@8Dhj zy{%JL#}n$$1Thqa)2^m^LX!&}?tJ4Hig}AmJ(<6&H5lb}Y|m@&V!INMm`K;d>-a5? zaGSK|aj_#i&=GMDA z#O@Xd9wuhNS?$>sjc4Fz*S)L zgLk(i*xQ|!O{+Ieg26%t3#oL_*@P(B%18o?Y=2_MxHo0d+S@tShRCGOkqmg;$vL)jPLk1Sk_Md{{Aj;JR0{fY0CS`{=c}7lwXq0?=ABZxx)tWgWs!PRdei^ zm%pq0H!Ac^@f27MFx}|Nn|~zo`!IkXpFC!qv;hqe6N?LyBLDeyIjIwKu`}-Zn}Z)F z*i#SxNsR61z}TNZ$1v1l?*{L()f$m3t_qh@sWyL=ZEDO{HX|U#j-0Brm}hsQ(j5jz zfFK9T!P_5LRJfu_D*-s%ghyz`_QVDor)G-?@UN8F%0P0Yk4MTTL`gv)pc4;znQU(u zLeAj9tNigmXfg7n5-B0-GwWKO9`CPrWViweD-1Rc(Zm}y+THx^q1V~VLP;F(X1I~D0%#mPge1&YtTU&10z`>>^wzLbfTJ4j^@;% z(_n|6kpnos1b9wvxKu2-k+{2%E?1r)ovGN|Bb}xCaK9$g{oh+S3gTlYK#MY7dWBB# zut#&P#DUBbqZ14<5WQ@}A$%#bzT8LLLURb8LrQkC3Yfp|0=uEjIWZ)7s)GVAX7uCe zQX=ZcM3K_)6d|Gf+p#9FDBD?X3}~stjZ4MV(a=oMjxr$Ekln|9q==gVw3d)v*E=h#D@JazI>fSGuw{zU%TC%x-+jnAVNrR)Zj3HlHiuaXEv&GY6Os;K zyX^5expZ)5GA7)UEVo~R44`z5BYf_m5xi!o)j{^&)S1Yr@9T1jsFfG3 zm3oQXVlS;7`y?DI`v!SkC)6!OBJAA++OVXK!PZ)~>UW9K;h{lPwsF8LvvgDVUMpsY zmU_oglAp_5Zj<|~9L2xXUW7q;V{9>pnf&Ut3M>{D%3*w^*;T|$!4>Ylh{i!g+R=$J zC!>7lqj?$QbXI^Kyo}JE=3hiiG4TB;S9VF$E&tNh=I{8RpSfuYB=(MqfLboEvn>*Qab1a7Y z7Oy|-^1h(tq!7?$r@wQZW2QP5pA39T_$%X0Qed%zLRiGY?Jb zoO@odsRBYjwS0XhMH^H;LsQHl+gP|Vn+YT-gw;$hDf@znJcA5pIb;!*)iVX!8$ed% zJ_)Pn7G^senmHo(+A~CG0ay-Q^vGdrRMvc%pBJBOtXV^WHu;|o zILw5TiWQ3g`eCE>{6shn6bL~sP{o4|#Q45@#LrSeh|DB!(98HC*}Vc9Xo(vp7%t>Y zofB+`Afu*{#$WMDK=@U$!V7|`%HSavfPi#%tl>YC)(m!716#O`W#sd&Wb4}r9tmcF zq>@H7DKo;irNOWhFV@wJ880~8RYGe$^JBDa8rFUHlaH#^=IbgB8124_Y< zs74*~WQ}y*h$VS4=Y^5VCWSkFrr4xL1QM#RsP@ITHyW*g553YKwys4l*}mI*ZN?m( z@mpA--HBr`P|Swjv>HS|{sfIQS)%f`^U1WKxk|l2c|Sd~ubjzf4O|nMe%jOlR{8$~%f**`AbeGF%CGh9C_94m$8J3fzvtQhL2W_g4 zJSV3?V&CIT!`GnVNMfGkF#O10+6>zeFRSdzMme+X%f%ZVzW4xnB=xlCg9x!JG~%9HeexcpcfZONGo<_o{aZdM+8;$ z36`ltq_?yYDQvJNkPd6H<0<=qe1&RIPrF`1!dc-`#JZ0oUNkhH)6)YU(e0%fbCxD8 zlv#a@!iPjpUMY1FE35QMj z{N(|FA_dXnK8P>@rq{1$b$3abaZy~b?`7$_vi*aWncv4>+A^nV{K^&Fiwn5Hjlfto z3F&?j;*eJVwlbf3Y{3Z#we7`vM7gJRUu=)P*@5FF?}Xx`PV3m=I68z3!yMslzOSC) zjQ#@L5R-vc9Va`!v&q5%Fc2FCfj=T)q)F5Ec&qs8m4Tvu%WP?70~$CLKhCVeF^Zrn zhWsK@?@?DTaSOW!VLBR&x->iCH$p!jP1&vk39ApChcqjAue$aA!#)Z{M=Jo%YA2oL=Hkv0S}hySVX&9 zf>No$FE=q0Ya0KNKws2h${C74r8QjKTuzMVOwKEAU5(I#vL|sAcE5{AXR|^nN?v5Y zrzU)YZagNhF)B8Qwe96r>G^aNR@^^81vibo07013o+;^g@B$cam~bOvvJ9Q?6}bj0 zbz(xiaL`UTS1k8oOpmi7vVK9+=n^mTL7YPXovQ z*$~YWY0gO=Y5$BWGWsi3yt=DN3i%MDB#tvPd&066pjeTn(DZ_2K%3p^mjNtv7l1CY zanCdmjK|danSQID15j!g}2}W|bo`zB$7qzmziR&LfpI|5^Yhr5P zlx#&3!PCWtKPX%>EK7ohHFF}yFH4uQS-`e>Gx0eD@syN#(v3|(V=TQ*!FFAgA6Htg4Q{o{Q9!wNJ6UoygN;TUV|YXfdb1E11Jwnh`Uz(;sV zgQ$cQ!ElVNG>5j9P%5+lz37h%DQO4^6*T9Sz9-rz`kZORz@H2ptjb$9Le? z!pF1qj7{2c$(mI3)X4cY7^>{%oZHL1zt1mjh!435oL?JaFNi~Cnn2SelMT*1aTRF` zKC()o`n2*01>4$KWz1%L4%VSyljaO%=}~JmIaQDn{$@uVnEPK#$Qz4Cf3c6478cqp z;qB50>YOl@w3ySF0P?AUSS$iFN05$1?Bru1wh&opBE^gr`yymS<;`?!=4P+Ij$tUt zAym&=d6}?j3mUdE*A${w?V)C|pI-yksVfVJCa_|6hVurZUhM;+op5_XNH{-bXqHrF z7`Qt)fkhK-<76Q-8IIZhJX-|Cu_}|zK@V#nyZGzQ-?n2~LS)Y`66JbVT(tVKB&S6Uxvji(rjy<7SNBb<^ zDz?DCOAcAO%7%2_ro{Z(;MWCAI?XID%FXWHTIpaH(|ZHyxKLTgn5p**>p35V)0Vi0 zj$kSR{)s>H84#!mRoloSA$$7*3Ae-zJ*Q^yt@Re{G)|fsq^TdY<9Cx%U=j{#T)`_a zdJ+DFrg37Ta25|9DeB%fMP!|VmxZ1wN7(ZdbOF_)tG1f1*$mD@qn19srO!q8e-nxW=;(f%uTFoM*)MJD z92LyHDo$&rXWE$g^PWJVm??yK(qeRo&88IzjX@o%iOpRFgQ!QZ3K~-R(Z>fa`HD6y zG=ut)LgC4RuDwE}UiZR6S)7kyHb3_O*#%iS>q!)JiT>mZ;hTaMdi7S?e%yLsN&H$L zJg1@#yQ}_|B?3=Dvso`GVkoLc7*;N9Do2b#92Wk@yWC%{HZ?&sggnX?$+AELhSY7(dSrFq8e&KP zSU&9*wLn?>``jWfN|e8=i-p9CE*44k+cxEV<{ll#;eKzpeH{kBE3mJ!pSDr)jJHMF z)VsTP$Z)@*#W(f4F-XKsxM*i>_2>RJEZ~sv z882u#1|MyQ+|c#uzg})?(bBJ$Lek&!G~BE-nkfmDG=&-1A_Jujw-C}>W@XtXCgHb- zL<)*@NLH+6L4)WqIq)HUQYSi7S&=nW(09_F#khiX?$Z3DYRgc|v>DzV{i|&h70}XU z@h=%1vEq5fSwu9P_U{sh|E*(Wk!QUROG8ALRT-WRnpdw%?7QtxZBrd6_t@AIT6Un& zdk&Y*x^qDU>C0WYgmm!nW8hP1x|B@*=B_=jc_|}^h?ZnOUgEoHPyA`FbOGfzXW+47 zVQ^!rsEAsHyAbJBu7~dV{x}uMM;jP?ylXq;-N0a`A;Do+HPc#e%l8Zop6?e&g-NzM za-eT!B=C{8hCgbBT8tR%+=j5e|%AKq1@~yeE`5!n&U0kG2kkgFU=5CjmLd8?I zPLPuR(lCh$Xr8m&I=Y+PAIJ2pnE-xT>Ma(yZo-zX`nfOmDe)`*RnakoWhm)Iel^T}B3k^N zXK=bn7g4D~)ReL}Vh1(7oqp?Anxo@eq+*xGEcMLyb0La}rYx88MDTFZq6&3~6t-0djGJ|TyfY1UNtv)m!9&smulXhj!ID{|(?ab*v_ zUgAn(WpwxySgM~#B92`OSJ>2atuuMlZQ^JD(_s>*pydagx?%1DS*| zf*5b|EN}9@9~7c8M!@nC{YMa+4sjT&Z=%Nd_6>>6tA87b1J?FR6eQ{kx&H_n0X##v$u#{P6A~F~% zvHSxqaaKxkUV`7aunuoTk7ec)Kjz>#0#W2v77M4ycxWOw~1+)M^H z&J1$%fq9wTDichqK%^(hHVDO*@2T=?0 zFA$9R_==S!K4+yw6CR_(<;$o_nW)Ew;M*oBsg@_BWq?gR1Hicg(D4^p%WBe5%w8g7 zn@!~rsu3HIE{MGl4LL6QEB(x_TKkdg{NGrMQLxx74^GEL;GDH$v1`zDq@^ z3#Yh__kpPB0jVGeiLIPD9xv>h?k^VntqOUNvCOX)uAc*GG|-xQx!5mJIv?}q)RBeQ zDbLb{_=PJ+789mc`*KqBd+Jgbm6`}1Gx#W-Oisa=Vb&=%mG7^Mgc7f4-b3LZ`(tg| z_m8D&;BtF-vozP(&_;v!p;HLdIu)B|DG5n09U=?lX`V8__fY+m5Euq(t|eHA#`LXc zUPperTlNHuJ}ghciZA+A3q)MdZ!bvJ(u>5}V~0wsO)3lr6z!El+rah- zf1*D~Miloerjaz)stDT)B8{l_Rey$MJ3i`#rrwS}Lr*ix=nKk5OCZW1LBf|DezpXh z7tEFwrFYlC8RxUB%d2$iMS~WP$vcJSXT@Y~RzbY}k#4B*3 z=!088HNeq&f#ZM06?}7jMUo0{E_4#6> zfz)}~XT5uMIpuKZHHMKdI~WU@eQ}xb;Y5gyg_9xQUwP!_w!hU<&3mH13luRm%G+%r zbQy4wF38G*ANvJ+Z`PY(vNkLq$p+8hUV;i-mHkghRRryIr!;4S1XIck!$a?5(+3rd z9{QTyZ{Sb0qxzdGw&YzmiAGo?H7O(Tz0%V08JcD^6&VrG`wCy-PUKenu2Ho^+qS5P zra#=vvI2HHpnl5H7Ok-_1aWSMcSJAKYqiFBWp_>z#i1U5HcAw1Y;S8erNk?&^&eD# z?jjjXrTbx2vgjDlo{=miu}0y@zWB}$)TKZao2&j{ZS#eheH2Vs0n5Y?_~AsFnjo0~ zvGPIeIH41e5Bl92Rv#PbQe>W;oW+BaOYTg4^Y@5CLu`NEKgHXcZ}k)WL~W;1*`Cc$ z!SnG0>cw$-Eu&>&Q+toKxpK=5Rl8a%gD%)#j6Iijrq`NnT6kTk@CjlBI3=fpWnsv}%N;O?u zUSn<91Nrip{jUstmaMs{kCQ4nGFzo`og)uC{2{Vti^eMxw{k7wLXqp6#>`2vc~_o0Pu-Kl zzPp>-__7GG0>+61nH(9q65YEMfg2|2}U-Ze{4(kE8+BNcTNQdToW>SIH z0f10{WoK^Z2`vfw;RzvjJ<&RME)W$X7X$MV%vBPUsHEO4=5oy`M$K6q@gQa*nZJU< zM0JgHA$cP=)RHgDyprR?CoRp zb~N}%jc3K7zhpa4Fu{ptWLI|+iI`3S@8Vu{mVmsi+zW@BywN##9F8Jf9y<%FYPbUZ zpbZqs@ptkA=8Ck&>G8Iex|+p$F+Y8siTL={yn!pt#z6+^C*|x1g_8N6c_bNoa%10w zIR7Z0{X4o=zH1gWZEH2({Wkq{Y|v`5m>a`cVk}bRrI2S6NMl~?kQyS}@x8ylU(D{< z*F8ssL}cu>@;L@-bl~bj*!8gR?i&k?rE-QkK#M&8%G{BuG<-j zY%Uyxa@<31sm2wAq`8!7f}eU`4?XfW4&$x!@a;gd*u0wb($o0yQLX0Up6^-B-F?`4 z#rjbGqJTSOu}3(+sj{;3D)!w^kKDRi{^P<2Ax}tGuRw&LcB7UaqEoy3c+nf(y5xTg zGX|L1?vW#0@KD>kPm7x?D(^5aw2t|mo$@+yLN6((%gwL!CGm%@t#uS~+N$({{BMcU zrxBiL=(9^6T1r}V(V3y``nsz|y%%C-!rwYt#=X5FkvM!3kZw0?otJC*ihcgy=<&l6!Z%%Oh?k;_kFPDNT#0{?9XsdXoPFpkr^JpZNq z^yAa3sObB5JiZsZ9C`L%X4`PqmueN3VS2pd{(pFT^uS+!8<(I!9fw8Fg4|Q@pGn?u z<3oE@9ckKnS$yx#9giO?8i zVT&dIW1z3cn4H*=KX`SJOFrb>7$o&P8@2J@+BZlLA14$IyI>2OMShx*GeRQ3=g0Qr z(Xh09cpmzV8JfE^pZmrbcNH2j3xU5e=B^z>+&LUS)IEh&Ji5kso~gr}(8!nh$k$`Y zQ8e#cTi$p1yzj?&KcG<`ZBd`|QD4ST-_U$N4x8rre18rr#Aq~^9eP|P0x=GTVK4&G zdx5qXv2n~%48M#9ig)}!*>QfE4xV%0_>UI|oE#T8v-1^EfXqQ-Ej749#;^`{g6C|p z5_ViJJKVe&p`d8Ad4bS*%n|Ma{x4hTj&Fhw?i_i=w-ckmqc{#7$oI^D^FuFNIB)w1 zr1=P^rTsv5?8_#4ue<=Bim4tm7F&%v!r>OV7`5AG$J=6!dDh{<_b@q@tu3kcAneOpQp2B1T93<&VuA7cmnJY-X90+tZ)TtpM(xwM7eVW^?jPgf0YHE z(2*kxXJsH8GHKDC`*P1NO;j23dtq-&uRyM+LrDQ8g#&-*Kz;Q{iKt!y9jW z`Tu_6k#1~x+LSMQ4z08s={=L%%PbU$==`pPlb&afMX{uAL(E55yh9Y8$n(`!-#j=e z9mon(Nx^R(4NSEKavHUuWz4vM%_5hNdatQRxKh+cnMkM-{MskvGHVNGvqx?#Y>4o{ zw|-$nDPBYge*SXg%?48F0`z<{2rMuVdPjR*qhC#Lf-~P3h~W!5IzNw60T|8*`NxQg%B!=m z*9gtmEZyN%#Kz!oObup_oUJ{PsZWLtQIMyx>t$?23YLcsFbYk9Hq&=C*=FY*o7l7l zDgmCEiEyy;RIkA}3g}+s;gckq96w7MUk&xTdy96s1B{fF+CE-eRLirut_DZk>C1gh@Z=r%HuLlo$svF&RBPcZHgo^5s~rBk0^AIST4(q09U%p78> zU+ml*VPY3q$bX+CuiYZ%94u}$I574+~L_ZSsir@q}?PBW^ATj@FUzTLKa@WVdfE++W1 z$6IZA8#cyq$}(aZvC_SdFBWSN@Ph#Pz=yb6+_N%5;D0PMx;N1kfcb5YrsYCpli`|R zxm#WGoaDngPRol4yYT@V%Wr>tFYx}lv%}5}UAlYx=hUydVo(MEgZAM=`veX^kfOb4 zMhzyo1XRYv43ha0{1^=^v<=0__#ULuTAkCugW1OT$bB~4tv`;!>LBye9`ejkvfP0> zFCK#*eL`}s`ZL-;_m?+oaeM+Y9bcR_LvKK4B18S0hl!^z;wf3Dl2y5S2$qQarO0Oc zzi&2%wcKSkVu(7c`GW5r*Y(!%L`tf>*t42!Xno+^>LXkmD;8R+o;gFcNRW#1i%PXP zb^j2lVi2VAQFCh?btHyt2sS-CnM*yTx6ff3k@u!V&`R+#dP@C8~PMXU!Pgy=Rl=y&YEixTSCH__5j}Jm_R8ZzSKI z^UKGFo&Kh02E#hd8pm4dRq9ud8XWfX5g?ePkCLxp&&C%m1jtBuUtT-&@|O;cJe5Eh zy#()^mO&M`AUUAT*KeER>gNkPy|=i{)Oid-BG&m)GF;##U9tE9l;L8NppPF&l{BD@ zLPk(GZs{{E{rcRuTb|<^EJevb$!U{M?PH>R{iv1;SD}hK$RECikebN3%+!V+W$Wh9 z+RElZ(#4$WY>kIaCju<%0+kd@)_2GvKotZfUN|=!0%+zT~<#mNz7Md^r@h#VkJxL?vZPMX?R$YF%d6wXhWU z&u*7qHPgMfWi%2^z&z5=rke_EcpEs0FvLGrlk?1A$fb%~m&TG6_MUS!Ie+Bsf z486<61r@aW;ZAyp^123gWn1- zzuc5wlLn)POXE6xpXDN8wyIAKNOGcGnzm_4tj}|~xF{Xdlm!C2 zpK1AZ>x@jn2=itJu1SPXq0>yG{1;{q_W@O{hk4Ix`x#UI0OMRI?@Dki6XA0&3pzVP z+{4udfss#Z4%#I&u-h z@U<52f2^7b90p(z<7J_SL1LLM%FIF~0Bo7n+4AU4(P88d&j?sN~A|G(uiX>TQRK>uB3=T9&1u2DPil~}a1pAZa{ z7JVRNwK>a6@b$TUwc@}BS#hNlOMoSLP0!A#%rl?N0%9Ve&YT;Nc(cKcC5q!~u!pHM z7sK~Fzv$KHVhZB2&@v_~gA6MJ45-y$06-Lt0$F+bTp)vAAIn!EOEv|;P*SM5_co2G zx9YC6jWSgR+{&%Y6ibFyw9Y_j!b^%2xGWIO!a;|qj-;A0nHy4o-cxE5k!nCxU5eR* z>Ag!-BS7_>3-gUgL`DPab)OEo`0r1!$uP>t8OwqgfScfd5i@j^cCzgp<#En|yn4yX zIuoONdV>$hmz%M72;qSXj!~PhA>66n8{>w z-88auMq;|)`6*D|1DzUHL0`U!9o}|u`x}~5BCN4(_HgHp-*ppL3r5u2;Ph8Fu?PP? znpYduUNahk$K379sY8kfr6_jPWYv#+^= zJAC^55ra~NHBl;S%a~$Bk}^QY%@$u$dHNSyai0%nEu>*ma_e^bSC~x|8X4OZSGx|v$=0=CKR#FFlTe0 zF=tb%DWuZRcVo;|?kkm>ilT^g&6%r0a;0CZR4Vn8E|uEPAMkzb`}pqj*!Q#7=lyy< zp)pK9Kjqx$b}t0L=%=S23Z;n-|3Xt;GWBCbrLo8prJ;K^y@*(!MfL6hEw7%9M_TrV z1%9eHZ+#?3c92rU2K>5w5L}oROS0T{7Jw)dqP-;m5GHsl@2z?6sha}#l_={0&j)2C zwih3}S2!}9@C+RxsiDli0V!9b#(5(SqQOP!`64uwewVAGmMkvVP^b)L^%X*F)XeGU z_rAkDX5ZgIp%?VpfYrO%lq3%_qQ+#X(aac=tWA4-WHvr9(KfC(gGyr_ELlYiWIP@J^`QdDbNZ6i7*I*$GH z9o9CssH8TMjVM)n(`j?VdwnchsvXi_eoa7779FVpA>0(0at9%%2cLAtf9^h9_Y6rvv5p=TTpA#S@(3)v`vQD{P!!6`8cKl4 z^VmY!IzTn>qLGwSP@noLO;334iY>0S25y+5aY3yiS-B>CZ~vUSNOV~L#(}}qR2~K{ z((%`p@yRj}@|;rSqiPoz&Q+b(yBM@kS?j@p!5w#n9mCs^efSQ!&z({qMVQ!|DvZ?{ zn^%5~S%-S(wa&k(acpHBPV+jav;4{^{P=R}9h!JVcDsWDoIrJYKBgwU7K_merRf{} zlHB&art+@^E!cBayPyJuW7al+sp%v3W?ZNSO>d9cfJf!7$l+Y|Mr#@-|JOc@)Mw;Hv!Jon1C6j{`jNoPVM~hg@UE)>mu7OZLd~DGX z<-_`(>z;kBT9S%_H_R=qi8SLll5)Ec9!rJ&W8O{!pNc*R%Xxvj@_O9Zf+w^8M09#2 zM8~A2!9Akyp=S^Aeg(71EcEMx*LBfFPfO#ZMsczohu>E#ZfGj6qn00VvA!^3Z~o+*gIa$kJtf2Q_jG?Te)>>yToH4qtk9%L2*eQ66PJ?O zx)!F3#?Yz(6M6I34l4Rm zV1sH+%X8oW%MpX@p$kkHSbPEB&;$&{Whf(o;xGB7oreq@yfwa{Jk^i#l;W?RG@b_| zYNDm8w8}ma6#IRWy-?Az@?ua);8uMOxiO^H(Ic{VG2vkkF+Ba-Yk>TQT>0_WE^ya1 zffF%uK=pg&9{vC?V$mCw9H`H5N)|ZHe=E)^psQNem8xcb_H!Hc5T#=Dlh$=gw`E@@ zyll6+XcCFe3WU+rAU%c2?_}~OW#FeVPUNmwsl1k!W5ZcHUx>0Gv3RLB)6-A!xArp= zE>%?i`JoU=>eJ2d8PqL1GAoyNcuye5{B)+4Z><>(6aK>Y?_cjAOhkme#b-%R#5+gl z^@eLVCjzvitM#SC73TLHLckg?Iq$61uyV!Ofw{V$i=+l8ClqoQf_~J5J|}2--+NaO zDWJ3(4GqTp@K%k^1d5&Fb1^o;!|sg7&05EC_xj^}8M`2Ytk3SX#haDf$d{eS0KP>* z@KwP58jnpCV@-EmGj4ir-ecnn*EKL)os6`cC%YTax1MYjc$eq^U}Qa<}( z?BQ9$>R)3qqv@4^sZzL)f}~{zN?{MEq*lMN1fh8RUj^UtZnw_suH4b!DXLCpI^2F@ zDK{$HCb?FQ;b@lO3VK@ehO4sI=;p+rkK)rslfV7obB2g-8kOrek6tP={Of?EdFN%> zAI$QbeI;1XZ3a=)n!sI2VcVioDyXBYn6Oh@>jS2wN;VyO*9 z-xZ&NWlq-LTkyv1>MV4D{Ax^qs^7)*R_+p^cKOoevwD% zRy=S&qVvm#U%n7DQ{e8C`s&j(WdIq^RLBzTcI_u<7^I`KDOw4H@fTWpi6`!q?$Oh+ zb+wB6O>(>AkBB;=c4bH)Lv|=UUbyebMilwKIUSZZGHeIT{w;QYhj_=MdBqUOfqfte#Ie*a z{j&cWw^rqHVw=3CTJO3~Xia5dNlaD&^A4|oGl{+Ht7odE z1M^nXw6H!i>qac+iCF|mzSZ3XrNwTnp9^d5hB%b7#%8amn_CcKuIXRCnY7!SKuXxN zh<12GQ;$yabP(uEO&aECnp(FQN)a&?0mq`I+2_p-?X;aqs?P0^<9=BYvpKmw+zv3K3m-dXdxrrLQgltX$dC7~p{NV~4q zLBac&514K;fYUDZx#R8oJW<Sjj7+EM#g1Zig_2WIu!}n#>!;Eo!9Kf_5yzgG zU(!9i>y6{O`Q9_LphlS5P?ptOwA0wp^WXI^P^r9HZxduN_*HQr3*Y!sJ0G0erhWt& zmO?X#CcC40a5eaJC%yg`W|lfBx&aD5zU-jBo`u_WU65C#@H~d1vUg)S3;hKn@K4_JG zuT!$^&7N~8JuS^ZslfC?u%DrYJ_&BIwy&)g-xEPT#q;_yVgpR;D{~~iHD|tgjP;q* zfdwR{)!|#U8{3u2cHIGAVaL5eFGRTEKCddbZhct9 z!j-I04RscB!slAqS?VE4oCOi?L#dg-_4%MOt=CS8_2ZTgV2F6AE($*X#4V9xWeh-5 zh+=a75jD$qYEin!pH6|19Z+=&^zSiROJzR|C*hgfhHE$J8-b9)PVQY0FD^1E=9+_0!L-~9OKO)uOjlD2RmPUedaq-B zvY^TwsCA$7y+YTCBXJ&(m<&Cno;BMDtoYCt2E!HlW!*djl#zt6b+C4PP$~PT{OO(a z?eHYrEvWP=wlD5<+7wv8I&9H?@66U*#Z#5On@|!3CfIr943S4KCY zeWgm$@d!wg{PBJCp>V96m6)xi8Ez5vG=d=C|3raHAonTg@7D7y zM;TC6m|TflODY$s2=z(K-QA4XM$M-)VCZA;fNfHB>kX@9gnCE$u_fe57LU%sD@T)O zxhj7LvdL(;I%jHghv8nTgjG9atLfQsL_S=L_s&ck%Hj1>G0*^_T@=UAcenW>uj2f5WFT6>d#Z|m zQ{SC{0EpOAG`)QU?dgE+mQ%EOu!OI}EJ(-FuLpq7@*2Duk@mReg{wn1l^6!Vr zl8%)_sCd<>jMz+xXZ_flh@{pfax7y1;Y{=BBVx76J7%Qd7b@QeJ!vqb08$B)6b-`R zcaZY^cl6?LmQk_g1rd)OYQ~g;FB-?kW+DSn5ilN-Sy3THfd+?>QiT0>q&EuM$lG3| zBt_h-SF!KCy^{JNnXK4~e_L%oH?gPlLIIk!rsNx+Qr#&BN2mSe9HjBuJMJOh&I1_b z**Ax7M3`1ROWKVwZZ>le=_K$F<(Jc}2_MxkyHkD!C-LBUln?owTW_bpoc0my9$pcL zY?gr62CrYRRNmtNM!Yl~lg7u!O2)=+1$@Y`7AAWQ?Fp$vCP>&Y4XJSlDOAmxho-p< z!QTg}dYWpdFfqTmSY^G+YYX6(Ch-A(0^-MxW>|*AdR0~JgPhkTCp*@o+}lmdfxK(} zpS1g)>r-nEm7(;y!!#72ri#_sI+reb?R!eD?jf-Tk&+V&s`E)&tuz@bcDfm&1ERjzCQZ$kDSNlJ=Y_pWs+sy~4FI94VQktd(FI7p z9M0PEL}FdN^BHN6^YlvZ*RExz3R|(UXxl`ZxlhsHf;YDxL&W~Yvb_lpi^Q&jBQ@=P z(+K5i%M=%k9$!WZgVB}589Z!I-c`L30B)Rbb8z|O%f8X#RiSLa$OtAoJSq>F+ND0` z`(IeMhJwzy1#bMJjEQT*K>~{7-J+IV76(gV!^cVuU%9tld;g$QN_tz<JcDy|ZBx?N(QOYGiSOrU&Sihvypn%7=oB*x3W( zWbxBf^UL?;EZoJn)PodHz=tf-W;c)TI5>LKJzw!~@7Z?`SKf;M=-OFaq`=eul&wA> zug=%duZ?%|%q(8}t{-UnMoWKhz0?pTg0<@$nY<|HIs$-JVdf8eMJj?iv=-51OnWTs zM1u_1pZ(y@V4y&7t5vPR&}z}_i-DaKONV7Rd>RTxt^UgKY+K-|oF+Z;H~47U#Vaam zk?6kuQRi1(Zw^?&&^kh5UCo`@RkTZZz?6?C-u9Oqlo$u36E-(NKuxb6O-i*oeW{d}i?j5^(0Jw=?hA+!A@c3ex} zHdwXJm=8*9tFyjteL?5?I`dMy>Wb}e5O}c1@ZjtHb?TRX<#K0ZUEAvasx){9wHmD; zca=9a-UvQ0dRx+GhfP$r>*oxq_c|{ix2MRL6^(qIzhAjy9)j3i6c@s&EZJ^${8n3+ zdx`&tnnBHxMxpt*OiaM7+Mc0f|8nl_cs!M=zhu?B|GMky4VcnDpF-Tb&Z44H4?GK8 z%v{);5?ai^Wzl`a#>TqwQWK}X7o!qU->RVgJ*6@0*-+b+_MVt^gjL|0f;{!#OBDuv z^E&;5hrxI8uc~Fk+^j>-szeqlU=&<$pHq5&@M?a*CzX}AO_n{+%EtXuY_4gC{S4da zpDo^heR{TXS)8{mspr7q?Ry^zoH(o+&RLOO%HrL*4=Z_O$lvC721d}}nC@*cqsC@( z?}T?wKb5QO|`EROEo~&CF z!$maKKtA{1@mr3mk#SF6_9T+q9^^&9`~t$3uR%TbetUgYEpHLGN6$Vo15Hmmv7%7L zi`Bcb6?8MXy$fD5edlKF7Ttqu(bPH?jmoL1F;2*^#ffOz-r84&4dPEcuP+NboxvxW zK%_k*RsWlA#&sO`2tnC|zZ~4x*zxmW-YZONmIGn zn~`|GI(yRT_{4jVSD)Ndv{m-UNbu$d+N-p1uhd(B>rcxR#|?H5l}0;`o(-0IWJUUf zHW$P%*T3x5XOwA9>#FxaG*mhy%wYO9X`SI(pV>~gP3d5Vd8pcWJH-SMuXBh9s|~KB zdiakUURk7mnz(CVs@T43)v78R8c$O3{>&ibo7u6|43x64BUkN-4Pe%ZZf7j(aS{lHD?(^8-Fwr6LzIcMEU{>Lli?`AXUoC`BRYsF9+&7M&l9w-!3wEcIR?sNc4i`*{t3I%e=auo$j=l{hn7AvJ(H>tTRem0H1DZEBm|Z zdZVWKeg-C~$=UI}*6v~V2Hr#qXt$2%$%WC?wok`RPwDkHRQX;>e$rraN8ORse5aVT zUK8?%{ixwraXF&npgmpHD~>iYp7iL%Kwe15L5b<<+9#LO;PJF5m3gzsC;SY@_(%m+ zhsyoLODVUMXr_og17@?=$`7u-pJyzLLCn+OuP4ZJDU+&pT^CYMUW&T+i<0Q6cqk^% zAT~yPc1Ev?eB=7i!9z*6LjL_e+kFrI$^F}+pShyKz7KH$($5zBsgYN&H}7S08g2Hz z_n+&X8#)O5F##d?eHX_NB zpZn2>1c9UN)9;HOH{f~+W9tV(arJ{uheAwQFZF~Q!!FM*_JXGFnpx!YR-u5u4^r0-?nESYjWJ;WZZmo zN^7A`)k&;;X?DCi?x^2?21~cYz6QMK*9E$*UQsU3;qUv_LOqY{9eZ?OVX%I3hl^#v;#m5-j)A_rJLqm9Z{`-u_MgkSO#U1+H>Q3v zxKQg=(XOaV-JyJYF}!vpBY$W`d7<0zl44_R@7rkuz^QC!8ED190AaxgrS*XjU$xYm zSu6;aAzgIFYJPhEJ~|pbE}U`2RGQRqydBn|&c**BSRfayU%v3{3hrwAllY2C3JimS zoH7>c9*e|wbp#R$UhR&5(3yFBqVckYRi2jSZ*|uP9waI1%3|Y=GRc^fOP}`sAjLc} zAJR_wGkZ2U5d7Xu>0|3&Fnx6i$7YTC)-3F+twXU2~`xH zDK1i4wgNa`;{t+MXky9Mr(1SBtS>s7c+~Jr`BxX{sUw%iAj4VEEj__GDCT zRQ_-u=o_b_+;X)zOz}t?eqndZ1w@J@R@{-4~1-N_sLp0eB=RUH7xV z8q{xC3(^uoO$;RAM5wj(twrUta%=qX0=OTC??uU7x>=n9QF7WUpW5h;Q8G#oG~X-i zZM7!&Z1B;5-i9^}ud3tI%MUTQ0Z-k!0bWWi2!i5A%tNqjC`(+C4jZwx&K4j7!CnkF zmT57qT;T@G(G1+(f|9Nz0KN>ki~t88yfZF@gt617M*vNBdfNDHx4hH4v;b2f-?YZs z-Q}A4Wsqb6NM}HjBI4s&H@)lC`sV4B8hV%rq}&Gz6Y(X5fOii((zg&@0SOp^&Oz_3 zY2P~xbFMsbt&iB1{~mBcAyn4tui6mKjKBjyWR;hvwbFMP*kY|vh-eYy$fmABRC+kR z?lV%#eL&J;t9-+$yH>+39T$c?b-YfzOd7Sz-jpB`5OijK=m_5=PlZ?FfSc*x{zAD< z2##90nXCheBxaU|xYflBEKs=~>gU4^RGRzOkn0%oxRw!V6% z#`&9Dp;m71*LU)P9Dd{~fVNH#U7<7*K)Z$TZA>mr2;ac~JxDK03*=e|`9vbf@Eu`y z-2I#=DNtOx?>rE=BqdWbs3#a2nYgQ21lvOdaRbw`&kDy&A0%TX9VJ+^@DnWbJG9-zh2pPGlr?4PvFjsQN zVVV0M1|JeJx#u>dN?%<(D|V1qtH~Y*ilsPM2MR)EqdudiuuVJX*w?WSwK;19IcPK} z*_F};8VH?-x%!;>+yjf4?6p}HeBuo7ED#xaf~3(8HU z)&THu!uj>hEHsP3Wt3ZX43LBL)QHp-O8)7s-MWFxAX!?AnCZk;UGEW;Mu8}Ovg-yi zEn8QQ$h%tyi*DU>A5BwuYWp`&N#<&7Lr4?KLt4lV@yh@d0X z;+)6OuoMWg0vfu(gG_dle`e_l;Lh!IH!X0S4*0`bx8H!Rj0ixAK>a9xA)O0pnIOc0 zZjHzFSh{fa!5UieJtvAxhqW*!KD-;qOsEcyUN=3LrFcTODWB`kAnnIN|X zu@eB2*08V&m#_(lPfYQDA-Pv49v12bm0Cl?t<%Ga2rIm6Wcyg9HFy^e5hX5wn;J0J zDb7)-t+o9aE0WV>!mDuzALgD`;E{xOo*WCJiGt{Hg7qqRx2v0-6*6s7V3DY7J_Q!R z!Ac2v;UY+11*CRp)J$D!z$YkTgkR5XGI|D8_yBkrgak6m)2zXMlM5k}Jhu@z#S1X+ z;a7g+d&J!j#lxjiGAb1iR%=D(J&YP;u$fR2L4gJl5s@2w=N8}f5xyrSwAuneToZJh z<2f^)ZZQ>yo7I1kmm%z--)KtFJ?BVk?-&s zW43iTgmj3qhJJOyRLB3@yz1=v$29w2Y-jOZT2&TwJ zS5gs`=>$Ma|L}#Xc@JX%H)c>20KLEh`=EjYi4ZiSHLaKY_*6t>4_upl|HIH|s5LZT%|Rgow2R1BP~0H} zfGve^1tym$;A;awP+PyHfN#Mbo(zO}vUxdtzV8}0cgWVVB`{3Lrx`>Y9pc;EeHGY3 zhgX31<@5Ce`8J+>eIz$<#(s}@Ljp_SHXP$!tiFhT+aByg&i}T}aNeJbs9XA2whh{xgXJT zE9rt~6FDKUJtK1@5ubqGtqHQ#2f7|yyH?crrmziUMnnXRIE4dny&nEnhm2qb-69kR zZtxA)B)^jj4TSIQg0gSH9zY83$bbFNH}Uz}{dB!eHgmSKv7goMJG=K(zt z+$)8rTmfYnWJuOXVHr@57Wj5#x+w*+^5vBuGF^W%J#Zsy!Cj_<;;usgiK+o&AYb|F zXH$^8jOaCOjjIfr7#|15R;pd_`r>d;xk+eBD}HPgqW5=RDgWI5 zvac?CBu!eT($?PVdVf;QeAkkpsD=-&U;&|P1K+QJjwRpoKp`|aUgjc^@y=lK%o(`? zuyjX~bRjsjFud_I|M%~#P!9YW3q)t%)67}1HmE)gidb1est^!o6{W%3uon1tI?9rP zBYdSsP!yie$A@@*Lk9KmRW1U9rfyO#_d*B|wMLv-596s5Y#k`}QhEaMSOUKfi(Z3k zvOoCry?Lh~!<-8p+lZxo7Pt}IR!|=_j1MepN~(NJE-K)Mi!K6^%y0tl60-o&mX)c|(4bL5cklO&)@(ZFs3!lGtZB;nAxm=UQKcEo?Ms1g3P z49UxQS8G5?Z6s8+gM zi?V=|xyU^aVaWEsxEgw#xb;W}8}NMtG|z*=Mqu(O5XB$BcE$v<0vu)l+j@;7Cjq}e z_>cP_U&T4uNg#OSrRj5CAQB1yAnH(FV6MA6o5JnSpb0JLY(fV(Q_({Y4|Jqw(4-XX z{83uDkE9Ev_x$~R>`2U5nOS&$p5alw2UV7bUrt;+s{iP`T^dBzDA8c5&Uzg6YF++x zhoQ^{e!#1?n51Z=%p`uh427!=YqY8P`MusAgzGt?a3h;HR(zAyk$3spN7|L`2RVWJ%pSVkl-22LrXQNH?R>2g)Fk^g6Mjh&EA5ifYo>zM16^SU)edCBv#tZP!%1 z@!<&d%~tT)C4EQh<;k;wYQ1~&C4)zWg3s0oYr^g6J5F~@7}+5EL+1)ebQ4d@0xGmS zJ|@Vy?#Z_^9A4+^F;g?yOlXMu(h+GD^=L}p(S7HJ{4xV|2D1E2y*LO`;AAOPVz8j! z-&NX>Le;*<uXj@}W7Tm*lzN}Wq3_l-%OAr(Z!!}hMD!}A*T)sz$(oM3RC z@9TQDx=vJ1rbdDjEIjMa%L|cb{v6FF>9g_mhJ&pcSww%S>}l->YkhBMNqYMfe4J7` znQ$o?@%QIp8htOvlk#=GY=fWEsehYxrJ{+lpT%;VHFYq1Gp4L3nWKcjGY7#Jjd!qU zCj1uG`xeOtUM@nFMWkYTzIITdFn1IcVzi~AGDE*81@wASKvWuzI0=%doO#)YKyzv& zwv$cVzzF^z?;upcq(%r+VSA55RVpq;A`Y%iXIw*YgjW=rV=4Wb&rZxxp{gQp0b-}f zJPWZ0ghjyOaMctTnnIK?bB-rMuNY9;`w*p~IvNaXf^h?@O!RWKFXD-PF0wDKQEAwY z%`;mPIVo>=Q=%5xP3LcH$wQkjOV=OSL)AMc9hCSb-tIf$3UcBenoKsmMeMDR7|t(! zZGRV;o-}(C4|O{ueW&p?`yh!~86$?3DJ^dQ=fRV(BIQ}I?22Jc(X@Kx)O?sa-EzUT zOp+#puJWQ>NEJ6Is_@>FCRaHhY8ei2ZJ%UNaig0wmf5|H^;}IeElcO1i2Q$})NQpU6y}a6LX`U3`_$q4;ZAh(LC&AvEi=a(zZ3eVu@l>e6!h~< z+Wt7?`P}*#A%oxcIaB#l{2!Wo9&w-trFg~mLLO-l8L14HgEV_*sCm|Q8O0>7h2~io zXyv{8@T$*il!R8iQeT(WX|VJ4Sv0(T%~o*|z^90&<_rc@BduL^#h#FKB7U${hg1X2 zd*KifO3zzlWV;TxW6EF#mS)EdofJee*x|2&a893j=}(4)fqq^%DB);71q)x+ftfxl zkW9stsMdXVh`^^yHvdG{Uc(tk>FoE)4;GjRK1)Zi4;|HJfiybH8~D+ueS8asn(cT> zcca<5^}A5;V5T5=nF6iv2T5QEQbALfz}@c$hKa;X-?d152NFRn+439hCTMBIDN4{g zq~tk3@Ig>!cc*TLox!-aG)oAxWL}Y&bali>24J<;svdbR%U4{?vmN^0%gIy9iyx;vY8dnn+mRRWW>6%}pu zjvnGb)$meT#sq`-$PehSXik>xG8d(!3)2*dFdgS8`2LY0)6_y~;;=jkvp+vv?WiK0 zkr|}lF`)r#u1uSJD{~&4Kkqqh{nQsQyc{R_`fnjvTPF&4NJMkk?V91BnTe>kkyQF; zO7dpHK`HhAd?Wmc%&XU42&6BbJFa*HV#401J2FpS%X`JVy1&bF>~~0_2D;+bc^JCC zg7(a}=#F{nG7(Q_BBCQ?WFP24F=IlM`1Tf)v|YFz8=aeun@2{hTw>WCR82f z8XupmAgAcdZcGd8XjULTERV$x^>5ZE>x*(HP8j6zElX?Jzp|Hqp*i}GC z;JfCmO`RE$ipB!qF{CxGJY=m;X6*F{22a5D2pUG6-kEsKysX5wC=!pL3K0cDbO#`y zXLYe4P z+BJAyHrGPbg%n$ruOGzq&rjQti`D3;S5hv&@?M@;v@W5@$+rHKcpGVP)-BbAsqsQO z=<0~dFb*d@DD4gpW97MZ@@+!Su`kx9fWww)IzJl=?4kj5=wsGb<#&yUkWLCo-NC$dj<|!gy!Jh z_V0&H7QaH`d})Ywg7P83cmWF5Fn?ocI~!CRG1ioRb@ergSqFqS`RYnhhWNo%hh+M2 zLuCbiv4nRS#0;j@)4|x}6hda;;aNd!HNok<%U%|q!*g3rT1@p4j(^|{g>v>d{duxO z?)(4*KG;M|U}UCxzLZ9!4fZV!P=Y59aebP+9dQQhFeK`#j#yg^ z_7FL$VrpDZeZGuRWj6`W12GwVO>l^RQ~A*@Fx&I|gbIgl8eh=LzbO2%J&*xq#&#r( zEmXe^1 zLUC5sw`#BtWD%IZ9^o4vA^5GU9=(}&l8~v5(aDCGNIryTid|VD7=%=DiL6rN zoF8ZS`yYhdX1_r+Fk35+MvZO9bG;(aB!WE0dNFB<=3Das)hQq8B{c0E`A0Y1zt-b% z1|rxbLv?w!EKkjqNXg@DL9CHu=z+1DNGO4)Qv$2sJEY5+#CQp}m_J1&Po5QOLS=?= z28>hIN|TOr(#q*|YaSAS=ClXm*r>aoW`Cn0NMlVQSV$#vsCVawXfngeQ;zx=X9*Z3 zBDL`G!C?3jN^V8++cNF8X8(2+l~qkO^VO0j0)$OXIEqS6>`zVt!Y7f88Ng(L4Y6ZZaf*!pX?Rem^=yNOQsoy_uU2VB4iG|D-a<*1L; zlEZV2II^^JStbBj+MR1ifMS;rc%V-?+EZDer+#|a)mI2ju7drAqFWr@zWRCnarn2V zA19)04UZz%WltzW(C%EZ>MAHWv44AwtQ>5g8qw`b3y-kYcVw7qkqlRIrz&p}l&ir? zBLV(PwW<~n6Xt11G(-8gtaLS%Z~ZXl0sn+Mmq_TB1U0{JfUz(2<3Wxxos-P)z`btLXNvWPeeu&2}z=^D-0rfyQgGVfA zVu$vt|D*8Bw{EBsA}!NrAmAI$JBcj%QFN~15+v3XhHhuzwupfogrVG`K|4q;j~8{K z7E{*a*!^x>G)zloGLYv5E7RjoA?Y5_r=&w#XoNY&gEJv@_53hRl>;TY7rmSN?} ztm;IFIQnznb~R_{jv@PrZ9K_yXIe*0dmwTmiiiOD7CpBEuPuLSa9=huL2qE<=X2?* z{=$B4Yrdb_m5ePyrC|P|NkRX1-xqQv)I_^}Qz13IsXmQ=7Q&L0%M&`IoRx`SG-vDK z_B|y5Cd)Ks`fPp$oI!WeXTGt|$)~db-9|uCO^0?#Xe8=4%IK46pqjAzzwv3r460lM zR~I~hbq5M!h4?|9hDD!j9#tkqAGDcnfTKvGK;X_kV#;`12Gz(9fD!K7cLD=CJb7EH z$`a*60_Dv&u6zYWbB(4_0nlMolRPf1#Ycig@uE?rM7+`+k8f8125UUAMg;|HJ*d{{ z>aa%BoDgEIdF>B@q6DD*&1LzOKKstT;Flo9Kq_7>44(nU2!+8nUhgCUm{f?$HXF0;ljRHlfmVSrXpCY-UNd7lQjk)8=Gmn82``B3gbmKx zoD&n$83)zIzs)>cgjuVkqi)kBhBi8C)1%js`~#$eFICcw*$iF2U zu|KGU^1}S1=+mN<`#bh>c*pU!Q7r`BrYqPcx>ZF>_OPrTFuUQ+BO z2ap9HpTi8aQp<=f{elKq z#|{vQ&y^AtosR^;2K$X{sV1NSVk=L6Vp$Bc?K3j)*0TkV4@1;B_oY%G+d2AMp&T4V zd6iupJ4L;vLpPxZ^|m~AK~a!w2r=HrgvEuxK!^%pfQ|j{l*!hluL+(ay*3%dqF@JU zsxsmC;(^aWU`Z4Ru}a#$Ml))lUS6HrX6UWC_CbaL15JRs)sFNcK=3WInhewol&*On zvK>Z4z(C_{$?l;({oBCbE04DbW;WoDWQU^`ZrAs$dAKfDll;ljx69uq3-Mln<@J3( z4GNZ8?q}ddZ_FM({3Y-ZaNJ8CECgOhS&#YN8Oc8_AUpU!Yp>0nyws>LY~h`Wyg=u- zJ8HxtrHSCJzTj5v^7+O#G4iLU>sR)UIkVB1GNZTUwXD1RFkz7d@qG>Ik?6~6A8@&h!oXCB-i)F zMCZpzUlJ&KFTB3-t9>=#xAAaqncj7sw$k`l{vqV545sVzm50I35&~h3j%1?kFp~Z( zY^lt;De;*G9=#T|c=1Zc1EEXcqw24?b~pf*?9Yq@V@QyV$Cec388EO z$n0ZI&<$7U?P+n|O;=Q^2_i-R815=$iA0X*Sj?01e+X*l09RkaWh}r5y!H=WAz~A{ ztr95m=mtPpVBrm^iLqfMa@(H)nF=mbWX?zEuV#iDsUL~{y6Z97UAUB8{4M=?W4P<-<<1`9j2xNVc`)$0aoq*}fuzhI z3^&%CEA-~xD$_IFkB`4mhTfoOKee8SO9?s9bH0$_#uS+Nj+92-G0oL_2px*$x$!TR zIW%9Z5l^AL!9vS2CSVnvxToU!#C-A2UMs`i#ii1PcK9~kU%7&u%R0SQac-F#@1hEF zS}0bT@UHP0sgd(z;}$3C?xWf1CK8-qEfc%ioC6oue<=4r6t=352G<{|?BrojcBtA+ zp3W@rE$E~k&xz{sVy1Y~cb3$L!zDLx#z_24E+Mt7bS)+O_n}=OWY*=4+IElpS2rv5 zH%bOJLT8c+18U|I0$A9>GWItQ-{1V}w+ceK&N}S0`F5-KvCVnYpFKZD%sqQ z^tf2X?C9OKp|bl=CC*Kif0zrojAbtW=2;ylUp`rI>$;yfa{`%=^i`p&K4<5*xFqpU zzouL_f==d+6B*p(`tGm?`u8=cBGrpU3wd)Ig(FbQUMoMu^U|KIhT0s)=ezfB-m8~Y;FAeTy0ONF&-Z@Sw5>^S>pq5jn37n7`gJwoYQxzR#} z7T)%aegh)cyZ&~`)8& z_B_+<>qEDe=8F$h3zMk-Z9@O`TY3KEZSmiSS#~|f=T>s>dTB0G!%n>$u$w6U$8)P4 zoX8XZ&G|Eb^~=9s{`v3s-~aCW>^W%WId8Mb{Bvj!{f2C@ChhT@EcT$=xTmN`WNAG( zCHwwu>z;Ot8biaSy}zN?6MvYx(*nP^uHGIxVE18L8hPOFe;5C*bl-b-RP}rM;O~L7 z%~#^BrT8wYhd(YuA)D*F%XV`SA4ez=S>-zwhJD!uaXz(swrN-x zp=15BkgCy7-XDGAb=mdi!_K%bAD%SCYFhez_!{BafF2*dWAvjf7y*;kXvk6AAAT8oqMCh-#&6mDd8X~ohOoQ?mEM-eC(Y@-IjWsq@4GIsE5!RFg-o9- z`-YIDsN_2+lch)DF8B8?;tu~icy)(_l3jI5!mIlMkn-Cb`s?SE8|u#a=YlK_<}5fr|M!l@ucY5UKEIv2XWy8dJ1u^7c$Wxf zGAYcv_ViHOTCDdai@E7~pjG0JcD~lHo&*22{16=-G3}iR`A>*;F_{{~dbuZlXsP2WF4y_xATWDt)%#SIhSfvq zH`<UMq(+oP>C*92`}z0#`2O>KJU)-l_w#wb-_PeO_15(__vzq0 zDsH{FkpuEqT1+iQxNjeD)LpCH`iz_{AhO4;L;AYjKdrURQ*oC!{4mqvbuIYupD$va zcK#2m-S6~lE)YyVhvgQ1zCSAXMICVO@BQ+2?7~T9P(tt5_md5-H6DxV1?%%&$KE`- zYh`m#{>&ah*w*LUKfkTbTstC8F}MA>@%CZO;rsXg{`c$K$2U*!-xmk~LRdZ>A```D zz%*OpE)&*xy+KW z^o2%uI3F>I`}b|^P?|*J1pXK2>-lnJhG9!xRSLor4l3m|l^)zaI@t`>UjJ~lZ6Zm^ zM366lYhD$CzFcqbT@KSF-r2sT9ndGFyWzsU?Tl;*Ni^EH@u<3lQ3(m(ay(pNc5L&e zri>M`9{(Xrn>L@N+ri3{zJ!5Z?SZ_uNieDS)Vnq4N)TH6YC>uVLa_Y$cec~1Wr26o zK6u=fvAcF3wp&+)y6JDfyQA4BiycPz#u}bod*-@1qI6P;debQFihPe zuTTFJZ2o*h7dH6CXUWF@T4HN;{94X8|McrE$FZYlwAlBc@oHIQ$9^ZY;xKOLTuA)M z_fAEx>pnna^EvB5o%fLIp)rs@A8X9*I=ve6&Q5OX6F!n_I?7k>cyJGq`~AAEPzJWd z2^(7%2R@cDK+3p7#8DKDk-aW|ypjInd5!iz7Yi5@hK@%+e1GV>C-K0$ANBjs+CKef ztYM?D$669>$6q?c9e*fSuat7%brIqPvc9lm;mP~BKlZFj$fDRbkHw;XR#ir_gd_$^ z5j41n0tzBKIYR%WJ;~}<%=uKo6_&vcv#dYxRO+_|PnA-|{~6m|=avs_ zonVw)?Ij98NN>fc+!g>n;a2%DYxT1l0^7Lv@Gos6>~K*7?5*nKRHVYM&ytr~R)yc9 zVdRp!;-`1QoGtlT%n#?!2-q1qv*zwKMAeI?-mVg#ak+;&r%U%tX;^(`3!XBtc;RL` zI1RU&rBSU7E{I}DiOU~O(6IxO*n=o(JcEQSDHVPJ)Y{IR%5?))hkEaBP^8hbp2J12 zl%R^ATI*Crmyn`kcQ1 zSOXT^O4))rC)ENee?Mr$nPku8WV8=ROB}p?zM^nYbn4Fo%z9exslq`iTinc0{TK(^ z_CvA}J%Wj^1zfL|bU$*_i7Gk>@`QC)fYOFB?A9`&UIwbFmR_1?9A*V zvUmdA_(g8EN29KN6++^jo^F6!8C z7&OpP&xda;uB12ch9Z{UzJ8)zezpJ&ra^6VDm$nxGX*|y7hcf%y-r{_JD&Ejbh218 z<~B5Q3Plh)QOzh8e6l(+Z zC&36w$LUt=bsKlR0P34hLJgF`+`I#;kC(-n;R-i%pn1X`sEX`2Zy^CZtw5TEA&kj@+wtvMZtpJv&7GDH z7qAB-5-R!1y(IJ{xe@;t6r@_<&fWq6`-Wn(@Vh(zGKMaV=x9xF7#6eN_Szv^iHbbX z-pEs3Z}u2#ueZwmeEhNP$O=pApCJoVSbw@Y^&{-Pv{w&)dsiYWTdd9DD>|hZg34P` z!($f3G*VGlIjp$rJ5#D$dRYV7e$+q}eqERUDS>TUM&p&Wm@hvxy2msmF09EG8^wzZHZ{cZPPH01ASc!Eg{ z*@Zt(y;Fxq9_Fa6|4_P6YJ_9ir-pt=wE*7cp2>8r9Tc^q93#A!23zx!QTd$HcMTZ- zwFolcPjjQyTLj|2du}Wj<1+>JFXrcRum6)wNZ$8^ECCHn0eIzc?EafjU@u=Z0<(Lg z=*HWBRADwdrQv~rOG3Tb$r|6q!8#*C!q=! zN1BAi-%PaK7T2Q*&qk7L=r-}g$LX~mhfl?u`PO8( zf(M}2A`#~*3`+&(kW_Bc{iTB~%}FsowEk%Nrbs7OBf;}h>>nnOaP(P8C?ltU$ERajUxWnZ8fq}w8? zNZLz#JLN|bTqROde)9D3WN$56xcmhm6Ahn9PK`PVw;VnU%tFWGLYceMYB<-Tuf9fE zME4S^L6WxmB>dq6fHa2~bps5~5+>oGZd|YsAw4>{fDfM* zI08voi3LQVnv&=pGuJ>Q81$T0Oa>Lyru3gosLb+0;R9RE5mt1dJifF&t7KP^fT4l2 z`EXsnCdgN4UB-oiCz1H>Z`60+B-u`%D8NB5H0XMBfF2JXk350m5Z`1_L3Js%a(^B#Kzap>1(pG`xTd#sJ9SP*$f2 zg2IDbe{nG4>&2nY(_htL<#ebO4`SL2rZfTkLf%!sl9Q$!ImxK51Np5Hxf?BWFcOw!h=0vc`YYfUv@$P3BQLteO;1Pj!m6-2Ya zCD|8;p>N=@i9WgbT7>)qkS*8iq>OI6Hk=}$K}j^|H($^r=gXU2evveIIu~4xhsWZw zaz+nY5uHlV!HY*Bc4*Gu6Lr8yMCB>at-jOB{Ib`r>_~f^t+Db(r0?r__;ktXe55al z2hXlGBsQH&r$fhd{6z65<$Xcd-^Ys90xxe-S9ug6GW4_^O#A^d`w9Jc2?CFW!PJ zrxE5(2u&p)LU2I^Xqb^bNx=000UPIK+i5An!)0m56}&!giyxeaWd7EiauEjHDMR5Q zki3|d(NntRWbfy&4fCT+0r!LeFB~l2E|24Kq7R|?m&V8$a&)qNdX*?KzB&rieKRFd zBXRSA?c!|1#LMcxUey~?=XNkOTY7n?#Xg1E10*`w_SqrGGv90~`D=#3g5{APi9(H3 zmH8O}NEVz8fNhXLC3K%3BRE-;dc(P{Mu0&9I{R62{or%*L97qoS$LxXAjjQZpJYob zi1#4ea1u}zjj_cG8`4f*d|2tVEl%gxX?eFs8bKO)kf0yeuQSLBrb3`jP!gvTMLp3{ zY1pLed&9Kyq%)F8E{g<3hFv9#j5aiWrC4x)%3Jn9itT3R=MVs)MjH4t_#)w2&oOP5 zIT~r^O9Tm~y8k7^ljvk44E%zwm)4(T4Ok^<<(8#0f;kCrq1>BlPLQvql)BI%&j9p4 zM>f1NdgcJ7@=!x zFrk(*?Gc!YEw%wO5@SdQ2Qr||e4C(~psU=|h%n8Nviyrv0CT@wXIDhNCSF9!aXbtm ztIzbP-rkCbzVH!c{S2@i5kJ4(U8tMSxZP6q-~4;e{^C)Ji|nrK`< zDU?1Ax(pD0QGPclxob5V1KQ}Z%{?$l2L1ZUd^W!enRv<3>8@AoylN5EeAg3qKxzU< zsu45|_3o0pU&56AF?nS8XUaYWZ=Y|25F5VNreBo54Wt0sP>ytb7Gbk4bhVKO_479}y^kKSt`-aGMd zWOo90m)-WIK@IIe0WD>s$hf%LD0s>^imS1fKJ+ibh4G)?m1lsXbOb(Yn3QRt%ssRJ zR@qF0q3yQb!JcwAPmf2HUvtW}wa{{nu@Y3NfojZu#0U=_H*vg` zM?6Gm!=sX=&O^hmx1-=_I_0{x0&zfkak)d>A7< zXX>C#yC)q{g@&)CuV(i81vV$lH;nbX_o+s|G&(^0J{>Y4c=4Q``&?rdczF_brZnP= zX`ZHB@~!7Wa>y|{2Ql$7K!B{Hgcqp^bB^39Gvq|N{`C16>k|0RHRTl?SIQNwC3Llh z+D_(N5~>g9!Wsbd?b#=uBg5r9SWg_3ChfDB@jjf~ZQW1FPAQk8R_cF%9~7-hBCe(- z*G*l$a4zwUyqC=z+siv6(t9zaf}5MDO(C@0Nsjp;Y9KM*6-jQN!fL^NzADULc^Mkc zFcHqY5{+4bFnez;-);g_ps4R&TE%v1VA5o|$}_^C$ekYMue=f@8@?)Up#Zl#}pzTy|_@~>f{FExSX z>vMK>TjgiC{!s=s-tO4}8T+mv*Oa=P_vEt|j8ZlKu;O76Y45I$-Q1+8o%|JLkY=9$ zH9P*^Uek3iUcYVZ*S+O|kQ<|xT>)l-iMKv4GBr4-HB{&f%`(sJ`iwO~>R=vW+$xD38XPC$D%RU3{{xEEcv zDA1sA9yE~CjQx=O&E{0!CBwZVtGi~HLVLtf0FnZ67%*yrKv|+#s;V@2X$admYEnz- z1Om+)T*9KZJ|B@kZ2r%54gG?KnJpFUD8se8D zxr!LrRFZmJ;M&MMY|!^_;BB!G4zOY+La)bK)`T|jIqwx{LsHiji1Z7!3hapj!Ji@K z9wm-TD?^T`txi~%T*>P$3bknk^Omu$x=%HgFoumT#1T=bf?R=^po6+RjOw&OcDlZz z4_uJ8*!`?}G27f0G09V;ayDViQi?xzf1F z!62cHVdUh;a9_91wJ{b{U_;l0pmMo&3^SAUwYvP~YZ6W8jB-KuN?iCWNCxC${h{1f z=u9Y9JerSZZRhUk+RbFMyMGt!KP<&)_jUXYv!9lS2oKU9B+=|JZ5 zL?FMPsO6$4O9<;w_iIeV?bBa-ZGxmv1|~i<{TY}vLp~XlvXJ~UD0OBeeHO7`$ zjDf=vNVf(S*#$0p71cJT>ZHAa5L1-PxDeV|^$F==xy51Ts}#T+efO@{f=gEJNr-#? zYvh|18okWFIpyNj%9uQJoc!>}{d)!1lsncCGyL#j+tSbkMNFYur?oLy(|aJ)GBXSR zC;W)wl*_Ni2qjSGV+l~3sEA1v8KF9Gm0~RC6$EwA(H>WYO#;T+CZyu{>|=6HHBANe zWJHv*U@+hNW}$_H0o`Uq?A1!6jYNSiiT;90S-JfYqmQYgPEsf62tBMz_#tDl`&A`- z%_o5&=0%XB-6f1Q>&NOtAxniWKN@Zz2Xv16p&hxkm&q86x@MbPT%qKp$NS&y&F;0k zV67=tzeOaRjlsY4k-i++js+K|y8E53r#o#P#yqp!`)GSm0>3jBP?k5e5<08X5vU4g zpfkrTmlZB1XwXN2Zu0C~jtGqex?(87ao4f~t=hl|d5>dc7(ie5tA_bfENv{2u^ROG zs9%n+eq5Ss;5*OqJJzz2+0Ox%CBUN{Y8M^Zc!UAl5IlC20Se{kEX%n_Gl}DoL{E&rP(#1swlfo zCpqwuLM>WBU2#H>|K5yO1czU8qfZ+l5SYXoMzHRK{Bq*t-ugm%RFshYv|9a6=l zn#u>aY->-uiJRk=QBE;JI6m6+uO25X$46^BThV!^gh-d)<2Jf>MT8!PedyPmgvB8U z!(lZ<6EQv#Cv*^=U}>jh4MZ!BEp0Z(kxf%UdLX7u(k*wJMw1DG!x}&6_2}l;Q#% z{Q{q;Hu*D(-^Aqp=T$?C@Or7|LIdre){2Jfmd5Z-L#CdG844a*2vYYts$}VY9?X>D zsk(aMvQ)M5s8IJZS1L7&Q&m6gslZx|!=hn?Ngl`MUHGNNod}BuF9M4&Hd*1#RtW{N zy2`1nm+Lb!_qkM5cJIL5t9yh>w~{5SdVx}M_C-&2x*@(WU4k}#hvT$H?|uO|IM(*iqyY$Su02M~K+pMrt^ zO~GRhYQ#ua9#*#*LOHiYAOhKg$WO*BRpbCxz~$@qjp(!N7*fnJXVl=SyNPOJ>LPv4 zD2}YK+lyg?qcqwlqz$;WjSBrf`uXd@RgPF73rr_QX0UExMy5h0nZ}dN%!hR61`^O-Rut!? z_ZpQDh)Yz<=fM_B&7eox5|Z!lW32;uPD5}nRVf7*S7A;Bb3ja{NWZ5g6IOrk@XebA zNHruQCcb5x7VfjfeioZ}t8-kHGRc^fm*7>)+02PEtb4G0zrcYZvE2|p?+~a)Je45g z(*9FupRj7dA6mJ>>71(9X!U)|mo&0y9Mv7f^&L-qT+Sr@QKOmF`CR=3BRcI5)%x1M zVmn`(vGtiN1uz5^KmE6h?TRD(oZx)FxDG%ltU*l~+W$UFF24qvQB;~N6V{sW38PBBJsgqE zWjl-={#+9{jNq$O>=q82obhZRzR9$PFA zNzubZC^;%ej089qBUwtNb8Bs$(F$_M0D4<25+3S-ZRD>(yyWc8V=cV>7;k#79S7f1 zTp7atz6kwN_V^oHYzr3zk*;*Q=4o9cf|E|)B@hAz9gM`ZH8RT@AbL@nja%G`Jlyi;`@n%aS67wI; z*gLQpQ)s9fN8Xc`oA%1tdjLe$>#*6$wrHsosRcJmTq$(rh#&d1@*9y7$xPi# z8qocQg^E!JE;Is%=aJ-=2URu~_M+^)zQ`y=v&t?tp9t(%r1VmNQU|=uJLJwA((7HO z?nu)v6_!BwM*(+rfj3uJivXn?^T3YNmcLS2?ekPd z=7MwLP?r!6Df~ z5IE3a#q>IG467!zErXc@;MfFJ-H;rs;igTLJ=E!Pf$MeZceAh9H#&N6K|Hr0k!qsQ zd99v#;8H!nxb{Jz(r_=cHYi^@afh9aKj^_5`rry1jlNqZB{$OWsHm6qqZT&Xy|*1V zP-X>n#%6pv1&1!N%*U?(YSSw84~>5O(IG>cz;MDS=j|{Ee}k zTfF8$X5PfK&H03ib2P8SIzaY-Zekrnii`9mC>yI^mU~R!P3dgFN*?Vhlzat)IE%FB zoc}1NS{GGU$Sp#-krK2e&m47L!6C4EWt8b+7kjZrv`P(|@o3-@*uPpY=|&;tl1u6J zHmcH`g#t+l<1vnqp?TlH72aXzxt@}7ZRPaeXam2<+l6UHVdX1fPS<`-FLh39#Z5Ga zR2f7+>+R(Dq@52AnvR(B8s;G=lG9q&BKdsR)^@E&HL^&1*ww?5_pS=>(~_ z^K%8x`RHUHL@k5u4ui7wCfrc->?xbgxq>gV^yCZfi{47VmHaL15TfO;X|vh=`G z|KDkp!klZtOGAREzgkO6eO225L<6jx8ZEVJMz~^k zOlY*AQ+XkQx#uEAcGOg~!kuq?vu#(de`}R*8l)`UL|qnLlyV5b1TSOWy}`U&2u$?9 z8J3|t4}J>@T4u_kBSJG)^?d87Rm!qj!3&}X%XbRO1&H}jO|x?2tS-Zzrm<6RgNomt zV0h@SU3iQLP0lNb`f|^Db>@WgvVQYhQ-cK!gRt{}xA>HR?o9w8mGwyahHY@Ua|gy$ z>}bmVFq`Tw_Xyc{bz#%?VI-As&y+5ge>O_IjJ35p){94F^`FaY5RBN2IRNz15_;)uaDxqJw3_%pzi+M8v&* zhy5pf^w?49)rbU@$V7wp2%-1#2IWTfvQcJ{C(lNv7DlF3N1p14jQ_V7+!2xfI`ZtN zNZGl_b7D~pl_;h`RK~$5mPb@(a8y=&RQA~@c43qtr#dRPBP#D+RQ{8wg4a=npQ6tH zi{czy2y=4?Nm=C_j4ttrE)9+@i;pgkpDOy-Q~YeUyd(P3z38eZ(U)I;$SC|^A0JsQ z7E`AZQ@_1-tolQ}M~p#!v`uq#)7hAo!kE_P$mWO`XPE$aStLDT%_C*)s!aJ$(+^#X z>#b*_ZU}u$*?%nd)A|jWsP)>n7Ejj0s^4GT{%}nwCO6_(y3fZ`J?lOxF@4p?lF!QC zi~s09ee8+!nz2LZP<-q@Z~s%(Q^A_$Mx`GjM?c1iO^1B~GzE!92FX5-csnc;d)VPU z`e4LN$~3#@9qjqf(=FuVt=B8|rg2!! zcVak@IW5D2&wOCoGZqTeqh%OdfK_2P&{ndm2C|=;83!s3q;!P&^?&{^g7LmN<}~gWd*z{7?C(wTp zD(?gM@t+bana00n#P`JtAwgziEj2w~*WQ6tYc>RmhsuwieY0~j_;_&jT`K*bfp29~ zFjnT9*-?fvtOpB-ytK&B{`KvUql7OBu=rn@)V>HPmBAVWTkyaJlV2Aqm=U93mq=#c ztIvrffC3dnGegvRQ^7v6LbX5TVAZDJ#TdG+YaU z(lZ>`FMO{**TsSjaCJXN7`i;BF0uHBuAWRUlYJE+Qvu5U6|5us-2M4i?~}4Ga1hAT zNWW83s`D&k`nTrbEsu^1MtjQ-@WGRqCFt`eZ9YpI2{1l)GI}T5{Y~7-><}08&8Vbu zDJsYa4c4?^>?nf_&>6ZkSi3w(kFO>u$_M$@;KUWfb(2l})|;;$qLwuEzinAmfiKEi#7nvg+Hs3H6Z=z^ujnh03hPy=wh;{%!La2Vf`G zJ!F@I!(KRq=^p=Y9uly@?N?YWo6iN0VFH%>{EnY5Fp7dW%!BPxA+Dicbg_Td06Y64 zMHveWJ{4fl%X*sks^6ZiOV7xE9_dsNaYOoL@6*Vj+;(b8sgsFWm{Mq{m+Zo)m>^B= z5v_9bvw`L~U*13ib0WH_>!P9E=3DH)EGlij@q zf=_3x=QAAXVDJ5m1H24l0NcR}MWlW?jsPQ2|FmPrxJ!fQB(*%-qQ0n&=F3<)J(?V= zWf}$Q=$!7I3<2m>)ni*)crZ0Jc@@umr}Jp5hCF2odhy{8yLE@zK?3q> zS~tY8N>Y04fUN+xJll#iH3~asc4aZgOI~xdj~4at!g1L@D{ggqQI7?3PJuwVPoun+ zKCuD`3U~UJ-9n1~Qi=X@mDvLwTvrs7(2Q^~(Xmat7P?Celm#kug1#g6lW%{RtOZxO zT3BK&PPCqY88vH|( z$><vnH0md-UdaUGE`bF+bI#QMuNR$pta0He;jhqdoHQKh9!>6i%O(8`4Ga9yo*Vl zRk19j!GYE`cJhZ0sab@1buY4; zt-5G18F^kvj-4As7Cfz$-O8aGtehP`WU=*x^DusBKq*kKH(?u-06Wweam$RBtr!*< zItJgZcN|AGn6qrvRehYKmFXLhMP@ucE>ZnYu-=%e#kEZpuZcv#d#1*2h4tIF)W%1z z5DKd7sx>Saei8m&#<(_zBNQ2*4w1G=5w;HFBLuyEY}O!i`oOXTlN-e4Wp}x9hwTNq zIVUwd2dmguB=RgE?Jmng1TH|lyqATJr>P;3KndlN7X*VwoCDlO602KzulHLiww56! zS)|zA!XH-c-rO4>Az@&cS&S0T*5iX!AhQhHVO2zSDobOH%Vc5c z!58n;p;-?uYnywO-R%@M@<`Wuo77;$)xXbO!@6(XW9t#4qH?XMZ&W;h0O5Am6`IWFuqfQv z&+f<@R$M;r71QZ(2c4};u|e8a42t>@9i(p-lR2y!UK0Y4P+@QisdaIh7Zhzfh6vpst|bUZvu1knyLT_3 z&4jUvC~B<3<%X6JvzBFilv;vifAL1}v|M{2`#!WdRezFtHf=NQ*kZRT;foq}LX>No z^Hl9q6+(B}xM#W87I1t;9lGU;puq|qeKd9)!h7rxM87P~HWS?m|2Geb-S=yrNnfS1Cx*0WPZ*vd1G*3FfsbZzAo>*1C zBZ9Q5(62yws#*)a7$BGKfW%xCxk7s41_V_dSF^INKbV%RP$f_y11$X{*GJNFR@>I3 z6@Uw@Z%z+)5-|%LzZTX6R*mp}g6bU?o2*R=V-uoXY1+EW3{hq;NTRv6>?2nTAwLh? zv4B3t@_C~65wo0cT5m>_2YUs*Kp8zbRrb~%aSQfbG1 zb5x2UZ)jZR_$URK&0j!9SXI|@DM-V{~x;&kJzwy#veyvCuD( z;Bg%po%VZn@;=B6qunSo5%;jnUfUyje4sd~!Un~(0XmAAfG13ILG@ddneGsYf);J> zKgYD|wjNrlK7E69p+99Yz6oUmSX(WP#`Of2)rLIwJ{SMS$QwgPJ~2@a{b5WH=_JO=I6lGkG2uJzIQX8cqhrvQM?4ke+9*!o-W|5tzjZAKU;22L!@!{8c=$9V zAiq?2ZxC<{BX6z>FPN+$T^_)4Cjg!x*|t4>flqumguOqYjvKoVAy<0kf2Mjo=Q9-9 z_%z6*Du)q=gLd%VaFwbI{bulfx!08kBT01II(t_U##W#pI}qq1Ph)OrIO$;yW9^}X zBR-?{<>*(rzD_n)AyX^*vi(s{^+i>)XbOMq#Y>{5`x}ecuVA)rRipZz2p!&2>U4l! z;eg4bk~mpPQf|%0%hY+8zuoZQ?JT#v{B;lWapL+M9#CD!jjdga_6Q4+4z-ZJjtdg!^h2uIQ7Eol`hVwqg z1;6>y!y0_dD`hCs6LOPvXk=pfH2;qNV0DX=M5~jKMTzt2Om7oNx(4EEqf0+!YFvWe zwDnv}4tnl4DKlBd=8@COxcpkWSMl69Z6!b3PB7pB8&osl$?4gkxCHboo|?!230KX& z`AED6qCClvILm%QnOH^!f%TrqaiPGSp*>5&3I;CFe%20m#Gg}dtzZxL4zya7LAUq` zRXj$}XT&B;+AZXjN*-j(IpS{Li&5gMwVxk%lt9K(AqV^BN@zTWrbj4HBPKLZv zxoMgQm>)xQht5D-BArDX4U z-ztDzzFa>MASDNqIX&X=4kF1B5W##hJa+FCIdHdsC5L4EZppSZyjgvFGgr0;MtU_vz2MxL^oVqbqh=!npiVgn53 zNs`|Cns=@L$mP}}+|w`_B(GOl8cvdKN*EN~%D1k!4{Vc5tdw&QMGJW3vTTS7#|TD$jdY5b&W z1w|N$9B?t=Edo{wm7p@aeT(%5E)VgxK>Wz}D`;<&MJYoo|E--E(8~-kyVST$lx@~= z;}i%eRr+PFC50)YE}iXka0iR*1HIR&r500VmI=#q1{szOv(K&r%0R=wB}98D9+ zG#{AB>94vvIzjB_9Qb!pqbdgFX3%*vazbLUivmgKuh>l1z0ZF$&XC$u^?1)+B$~Q> zXT#m!Z1`;wNPa8xM}C-lZ?H!R01cBXYs*>v&H~6o|2y;SI`WbW9%DTKf1+oji5^_o zhyv@qD%=5Ma91d-syXayq|^D`t@r5Pi*~2&Uf*zQT9lM=V#ax2B(=fiAW&Oe%!HV{ z)+HaOy1vM}Dx%YX)dq^QmG6=}@wNxmlyodl$tyb?YSM0#Ehu+uuF0^Lba z1eyS~Vu5{nS!yH7f9xF+{&pNywMhyY@hPbycD`4e99SbM!}Hzc=DfE=Loa-Nemz`N zb#-F;}8E`(n1q6WmsO%0N&i5v-;NaTTF4cqUAeF0dfH5{?9P0=S4$5#<%r-z!kubFMICy z6JU$+b^@6EJT9O*J+ZZ;}DWhkD*kbd8b1l2zf6UkCv0{H)89oAP;C)#v$G%wtX zJ7z?Gh;|=&fsD>_yCwDl^ZkT<6+vs@|?2OoJpK9mB z2WaW}j;Sj78554#6ADAab>TCi~NgaIo zPoW#{91CLyat27A5TMfl1~~v+chU9_+SK10j2forWajc`vcHTxr2#+WLesI1oXZb( zRc$tIKrO(^f0&4xS!0>uYd`ICh7IHGk>nKjy>GFsI*TyOJ3G0Xr8c$40y-T@df}s) z!@y&K!l)+te5`$ldOh=kpj6L4?-s~u04ODZ5Pz0Cvq<^!iDhoAKEEcrsDSrAufCh+ zL}=(fs3<-7$Jx8NJ`mKSv-G`o^U&g=<#VD|I*pqY)*mR6o$s`t;2)epZe-SjyI_Z3M?k{Jp0 zYK{{Ei1-5nqgBgeendBqGQ3d2K<^D#D1n%F^8CJf zhgQR+tZm#0qnD>%V#{mvvc(%;;dzXe{34UMIv*?WVEp;yz#Hu;eU;TFyiXGmFP_)~ zh|X-Q`mFPEY0-WjFqNslwj*)t|NRi;z4_&hYOBsLvGFhz$@y(^@A&v3HsFHYfLM2> zBFhJ!pX(^jR^0vZ`m&Vw|44JTJAR`xLDK%1RzBUtx9V$tGdt2Yd@05UukPlK`Il2Z zZwS*Iy?xE46KQys2u+`d-7|Aw+O1+*GSj{5Vc(YnM6%ldJU!GGHTpYe)Lo>>zMg0T z14kVe<^t6x9j>BIuP87y<~_vIg?-=B{SQUfEO*I#&1=`NchWqydfAEG_)WjyW3Ji} zu|!q>q?lYp*irta)IUG72H^i5L&z%V&UT5%(a(=eC7lMi)SPzW8hZzAkxyPj>y^JD zd(Xa1`7}ot_JL^qC5~zzACyK4cts?C^;f!Y-rO2Ez^7V_xCkvYCoibKstu0(>*w^* zm;ie%f9bVC_T8}cs|Cl-rIr62I4x{Z+k8PfebO4rkMb;f?<|N5{UAlC9e8xi6q@Yv zp>bt5tRPS%=y)I1c5N){>qE4^SD`2UK`+e7aKZN1oO)`p%DRpRH#U^D@xFwAd3+k&AH8sS9V)|=OMlmtinLU z?fOtngbcB&iC#PHH34!&wV?O3#(v{)VM3}U{SgfnF;s77OtD^f;0#zW^9wieMps>+ zx*8#M0Fzv%aseA#u^jD`Q6IN&q;`@(sB}Fk-8Gl$Fn!;Mi8lV_+od^`C4fwkUojqI zWUL1aV)1rb_Ikxx$g#=q)1@uCEuO!AJ$+m+O;Vgvt(!kN)s0IxfXrf7r{t=*N>J@F zcVVV+h!Iij<-7X_QcnrZ>v-!`pEo#e^740LX05io0=!@<>PO<=gGayq6zt^zpjA0S zw*w_vY{}D}uKO!3sq{>Tu|L_Sh?%#aFqJa19Oi!$6<8f(nLYyW3Kz!DkaQgPz^F5C zv@?FC_>XA{gp5-~&a?A9>6us!koFCS&b3VW4zOTy;YRx9WB8r!~rVIsamaOJ`H!VxUlFyUF zZD>AncN>y5s;;THRLz(5VHLY~PWhZyi|}G;=OKY*u8fL?Zl~1{cHF4HK|i&%y{UYy zTKVfqltiD3FHlZrt|ygxcOn^TvYUhLc{kkR@++!zL&!R`pyCo|4dNk?O&&h(awILNpQbvhi2QkpIVAA`qB9U|hth`(7S^Jx=q}}@ql$k)kv9v45o)uF``UZT zow`EZhr~tWi#1l|sqf3(m1OXm@QG*b(833U#k*Dq^Abzx8QjuaB8k<$AD8|%`#ZAL9I4VO^9<8?pQO7H)@G~^2cYdX9mm4E&$HiYKTb0jB$LgXN)ObBIdgpDSTGs=Ct`hG$2%0oWDi@!mUPmraxgGqD(5&^h$$YFX8ILSRhAHJ$@itbaob5|r3iCl zYk^8l%;4&~1Zwc(wp`X=Z7xAg0(w_2FJ?4fhXulAHi6~hywqgZTX7=(Hkpm0f5^Ne zxPs?{;Xq_yga3%@1+nGdG2|NW!8I<`j*E^*E!h>4h3E#d1Ilg2=)KH8iN=0+^D-DE zdgB+g3m1#&h5Q{M|M8Z}9AqQiTvZBfqws-zfkYFvP6iu}^kkq20qc@-QzW1(!BOV* zxCsrT2GJXYXeezL{?-^sIxw!*M#&f69Jj9!7T5G-UO*2=9rfUYLOZc}YTbp=7sX`m zV?maYT5*W%9zVJQKaU>D21wpw+lKMGGLC{eTU~JpGhY`+Oj&fMQ1;S?Z&Kc;7WKX3 z90_u5RApTjyn$$Mr{k>%&RA28Lb|~Vy$T(Kt&!@3Y>8wR?EVV|($cFub~wKvw(kqw zm^doIBo;eSK+Oe*NU#HF&$}#{lq5ev_k21 zA^N@_f9?f~+?K(2_ug{G-@#rSjS|9bDnKe-LwU-WqUibaao9cA@(61zG4Uu6u9-}* zqr)_diu2~l++LeC-o@lN==tKk8e`^q4KwAV=QkfxFMBaee}T3Mf0Hek8NRCx50Jq0 zB4m}$Y}bxm5QRecLJO>ZMF7q)|?Y zb(S)pvfx=15t2`O!OW9kv8CNh(4YvQ*u7uEx+M|`LHAql9mOB_7Hltq!nF-M9#|&T zyX3s+IV-a2COWk1tWux*OwnDU_B6Vzbo(IY8pPRP535t(b!+z(n$=57wIXv5JOjyo zzi&jZ*7wU0YCgK}X0N|x_;iqB{HB2cyH-$ou#0j>Vtd}tMDhyWUm5=mW$f@MF7=(~ z@yP3!p`J-l7UVg5l6Kai0vMNYdA65qK-yU*PVq2#SWNE^SC>_ebl6MJS8?6M?$#Sq zr}c59CA1+=LhRugRKyOK2vqPLK!s!sC(T<+FNGkfcZpL5N^QDR@2mFuL`cz2vR%#1 z>)kqpZ%ptLHXo9Z(hqFzTJ`H_Xl6Oe2eI<3-c=_hll7|pOpHAZZ$>sy@(V6oH55l?3m+Gv#$<+_f~_d@f`4|^jhu<_Y=j!SCV=^6Mv@tqZv{? zt~q!CSDw-~SmQNF>e8rZ?H@fJ(#e--d-KKy*wRksjbCwhRuF1wSDNW7W%9o@Np;X) zPABK9{k3Si^uk4-IfXq5-6+n*95R=vSP2mi)G^W+J(Hj_cD>a=zI}_{HZ=m}pc~oz ztl*t5eTjKQ9a6cumPvV&LJQ1$hwbA4tR9jjg$D_i{w_;logb)%#C)^qE(AokpId3x zyfhI$ZKmlDp0|Jr3Km~vpDJ9&DJ7m-`=CElD&m1uiVcyRoi4~KVT}WxvcF)N{Q&(X zSjq3w{alwN>{pGJF!}9fwNPMr@|eY}K?+=Zk6>TckfeNhVe``(Hf#}szh@Y7Bq?kO zx6q0e+mTY(NTEVqZn)EXz8N^;1lY;H2*u2s5A?|Ys(Jv5Eq_xA9FQFG>vYG|l~sa= zT?ATenh+vWd0^t+2P}GfGNS$jYo(_CN$|Qi*hZ$VJh+)Nc>xN9JCZIPQpp$FA-IcA zz9FFoFKh#~u?#!P-9AU4@wyJ3z$g42Q>cd-W_bbKwol6eG(+_ly~<~S9iWc)=Y_-w zt$#*UT>BE=?1HXJVQIEa`~Y2Q<=&Y0ZB(7gD)0>T%^iT(6m9f55hvR^@=)*jW6#&0 zM{U+6v<%s(7$*UeipFuYjoqcLlM(c z4$%S0abep#-7AfnwfClGwn5x@FW=2{XQ@LXsAH6AvM4cH-9Fl&P1kyGx%4w!9|M*@ z7;}$47SQ=%S0ZevW>&l0n>H0aYr=xzNgm@+l@e%?3E@6*lo=ZMp5#O@z=;~&o_%VhEp=+o~ZS+(}=0sAz7a%;7wa6y(_HUA9 zBVnQHx}AmEE7j^|9sRm@RMaBHZ*I^fYIK7gtnqK?L40Xeg;pk4`}o+{b(~Bf3x4&X zHr3If5vpzWRnuP653>ZtkLxoN#vVcoMJ}skJ)z;}QE{u*z#1P9znYrCKH5@^Lhm_s zoRfZqy>8*MWoRA{z-P_V4N{H0pQb7Xe>G~7lIwk+jZK~X3%RzN>Rm3iXQ!5yBULii z-k3=&&G%L=E71BiW;$LI;!z7z>Z|?>(zabQ{JNnN+opCgLS5mUlHU0f0L-6ZT1Q-t ztemaHKkW8eiVSeNm*}Zj!GSA$(6(OD-pzuSq>{5ARi5mW4~bK)^-cWBVNoK+KG$fq z8<-v)9ebFs?jBWLIGQ|i`zbyeVf8U`JxX0Jdi7YXf9gA0aSh>0ikwCowbEW21SM&j z*)OMPUj3kD*#ETIq0AQVa4^YLw1nFbG9CZ3Om-vY6kpLs*Zj1?-|(rUC*U3a^^UWs zGri*CnDp2=_?z>892^!?k*21a8D&WAKKEGX9DYy1_uU#&u2+$c zJX0q#0}PfUo!?n#0n{H6(GZ`F;@g?i!VDwp8DlO=9d+kC6@wRz&Z@!0V_ z#k=#n!>%9BCEpz_pU@XJ&;-HNO*~z-7X9yIzl8KMmgDY&HO@)(fv#q|i_&%<^nT>$ zUuMGlz6rc*Vo9u!G{PSDiM;Ck&ef5{N0of-P0pX+NBrRUpFS|t-rnSgPQqLJc7M5v zai3n{eX1jW+b?Zz?munrnQiX*(x3juoC^K%G>(TM&{|`_-qwDEb7n#k2~C>WKVo+C zsZ3ZxPsgCUb97qxC;tfTh6q@Pv)=Z#yMofz(IuJ$&`i`!1EP+!L5u>T_Q*zmdm@fu zDvdk9T0OH0);X`Q>M4Y0jKl>x!(^Y^dEvbdZ!*zv;OL9YOXX}7oWay=*ZX9kh%1{wB-GBz77r< zZ~Ox`u4Npuj1D}kZDg6@bPbXm=$GD@S@zSZ;-|Bv+>yq{BPV{Q#?Ku|vdlW$n6>N2 z;R|xv-WDlW2GW08I1Myrj|Jx3Zp^tmm-ElzvP*JD=NgYbpF8?m?$~nRvA2!K&IKO* zD3|*!F!yI;?!R-nA~{ys&ts4#me@Q?lE9V@V#^VZBAVE$1dhh`XCMdC#HsJ)Xawm` zcmxOzXSUS)KrG5c5#KYt=XUW9r^P;HdlPIu2=<3@1xIDSm^2k8G|@|Z^o2hsT9S>kd~vB_Ahc?s>dJvknozzQ6S`Y z9{a0iOvu*0_Qxd$8xu>Om^a9J{bwV(XV7MsS$xpxxY%>GPGAYo5pG?Xw9Qqk4-3JOsWLd??q6d7PGR)kA(dBQmJ$2qBz zYJ?&Hu?4e5^Rh++bx7)wPAoDs(!WVBRaL|PV6=JRoMy=(?;-^jM2Z0p#XCM*(-0Ph zn)S`;m!Z|>Dwl go64b|{y9M+;)${b({CGzjgNr=1m*3D!mq?f&*SKm{b5Q(Od7 zu#_oAnA)d}N$(NQJMAgJkCg1P&-!sgR@u>TcfBK6C+f`oYhEhU6GVjj|O8}+K-U+oi{`O+NWRzPi1yTu{V zu4O~plm~w{o9cfmEibdS4$3bOr5y#*4*{@tn~rU`m;!*3Xcd1jY7{Slc(%||@%ZxF z;48D1WtX?AET5%<3P5x|fv$EI^t(hy2(<{8X0*z)lIl#qod8)=nbDGiMBDv^_Ul3n zKEhx;)iu$6&~6#C#HcT-0f)BA#|f{Tu&JcgA=oFh{#vmIg=ha0#8~I`-QVnUw!*7^ zS31kb(--BMK~_gO=t+&7?I^(}k3xJJ<#1-dAxQGojdF+Q&N3{q$rFSqfuamh9{fs!~+ry@Q9Y z%j>^jC4}t-lUTqt2H_!Jwd~i)iAiana5z>E@{D@%Q5*4}?8d$c<qKcVA%T1thbVT%X_JE_V!moInC~KMwe1n8D>eA0onzj zO4Tf$wk#nu)39FlAgG!`&SKgZX@91=Tq>Kl`iEvXG}=pLHRpXl`6`22H1T4ISa3%0!piTq0UV9A-#AZCDE01c8e{M3Ue z{jC&5|1h01t-N)G#*ZkM6{3 zu#VPEY91I#6lbZ4Tk%}(jU4Jx*)iexK@_>&Oyjj&^fs%L2gd+%N+7_HXoO*4GTGu% zET}Nl^-m3mN0i&-_qC^$skMBan=^7KfbAB3O$yN8&+unV5=u&KlO_lrt>nL7376MQ zl}5#z1zVG+B#!iue$5iIezx1NP|b_mCul?8Za$0?qU2Ut4noBLv{uW9fA|^@Gb)!K zu5s3^TKgoz+~?b2Y*+RMbP)FG!XC@?iGV0I`Q$e6-q+u+Y#B0vGS!vu;zO04z9qGT zb#~pB(QuNhnmnCoQ1`FUoMh$*Ab^PRTWn?c-^h*2jFMbUwzJ~< zsamh+`^2=RU6AkDaAB3 zv07=HxlqbIA%Af{^~geDdnWXhn>pju5c)HKbwpK3*03zlq<7L&`8?6ryj z63|gI0E)t0{|EXE6vvVrv;wm#f6Dj3jFL#em;sB5P=gq4g_mOYYs&fWOP|P(ZrGbV z9o-mq3>DK9QRy0UJmyqMO!I+D(=jKKBdmiF15HbArUPlbQsaSf^8lrhKN01|Ej%Y5 zm@jYuqwhh8blXcpZIe`|wwCDot=KSU)99e%(Xk2>{CuoEg{CX*%a1y<*Dh(P413>J zT?&F`ReQ)|)}MgdBe!uKCX;60+)cBg{O9EQ${1|4h|mbLn1W~ zUX^}n5g5|KbmdL1aAsZ5!f*^ObW5tjxUa{Ft$&~FC6K>A*!RRGwBqXp;}HhU2La|~ z!*C;ivd%j9@gwv{gF4Kl#QywtOOCaAY363Co;e=U0;8pGggZ_>G}6x!@8GLUdW4S~ zO#fMSr)v_vxx12aDPl?_9pTx*Mla&1%g5^=V;IN&PnF_`(hqm->x+Jqf|0*Dy>T`; z|Lq%!-A8ZK{icT)e&~yT@A_u=-&at-AElSf^Hs8NGD#7>zhLBB+yRI!syHR@)qlY$ zb{gmb+)aCCX5{A5{bx#yjRvL2XGG;*QcBZ8ij(2}K^rL-H${zmMpbxGVU5~o;>b1Q z6PacYH75VGr}7$sYfm)Bp}+5{M7I1Wql}Zjxy07`v*;#4wlxu z#{}&Zv*pP%GTTvo=Xj?E5PZxEOhaFYYXcz&R=O(iKfPU4NC>~LKx6pD{weqeY3#WA z&u=woJp$1Y3P33=wir|)-d37frJx&*wcCz!Etg0DtJ@KRIHHaC`Zz_Ri*HeW79@?d z*JW-<0&q4|l8uE?xbQGD5SYNHs@b+8!h}nxzhzF+Kt6QOwo+>t>m;p(Rfd?U$vWH{ zR|>;x;K+<)JO12OA8Buw+eFIMBOe|;6$$g}pdIZ&ilDW`$87zk2uZ`Wk|nDjomO{J zD$}~#mA@31n9LBf?ZQYXBLS7>MKcAYJ)vjI4Z$R0_MfC76W3wtho6?j z5IJzOc}LTAB5)qR!aZ?as=-A{nZ9K&*BupYRGL!M+EJRWjiqbsV3am{uVSXDV;3b3 zUGDv}jK+CS$UhJkD$lrseB;n^LCnICKT8Ix@d2qHIC-Xgn#_IbN!kv6fukoI)olV- zvRBs*Ut_~VSum4+t}NVxR$FPDua?#*d*o`TeBV1Ue*#mVCS-|`q&c3e3&`6q!X>Y3 zOm9oM%4~XoeYWx?C(GGI?3!})8cuv(WF;+M_32Hp<|2m@C4ecB@5`C7?(7;ZK}PVw zk}24H=gFt2gRPDvd2c|rX0epKy&##ULihh=!`EKs8{H?u9@lKgme+)sTLjoPX$c-V z$U)f@BcHt3s@gvVPQWAQo(=|ZqJ)cb+jvslP-Aqg)0+wv+^GEK3?_0EB(e3U2w6@; z2mS%$lKEp;d;7utQEBKtBAfsva#)p~HJM%X0b5gdw3(q?HeZQX&c!-7Elc%qos?c+ zi&~T);(fQ15F>@s>QQ(4r=U%H)j5~k$HOH<3GOh=hORnApdLFYlTdm5`Gc)Z%(4kg z`tM?C6kbl38b5AjHzY#LlmnQ;6JT8E*>gG*t8$^dZJrkw1RjsTq2{T&vNQ*UALBq4 zflm1Az=CD}|vrJIT9unwA?XbNhy%HwDY!E&ZEIQTU$B*SV+EZzlzQF+Ls2-{?v z*J%Vtc9|Kb2W08ZLNRfr;CIDzS}{{|gi>S>I=_=9bkaY?g^1s_w=3xTr^K{X1?<7c z@v#L}zyF1X#c?2-PzE7}TNt#(bucJ)f*+zFyj`q7;z%mSbTLo<4@8bt3^ZmhwHxk! zyAw(U>rS$igXR}$*z=7Layv7|44kdvAwVBs9X@{v!30}SDk}Tb$X(NP!nwVh2T$I-9^VbQVPT* zf2lAgl%^*`IH7o8j1@UJ+9qQ9XMAdtfiv6Wt$NCCZV4~uu5bu(?RRghY*ZbHk0!Re zz;Qlbt43SCsWxnC?vG<&b{{T;nVA-leMf;{QXXt*9fqsRzF_62A<-K_Gqq~LW@s!+ z)%DheobWb;HH{Kde%)4Uzb&VPyyI>4D=xO*9#qY_S5cucdJ)gKOc%n^c8Q6(Na!Fp z$!D~qyIY(H)uYTZ1rH9iWap^;M)1Kauw_ zf>CZ-z9i?+dSP!IYwEJnN*89zsm-OR#{N1jIr9XEqL4n@`$t_)h7U5v(u{U$+*D{} z>KLiA^^_7Tg#~E9HR1uMj?7y=a-!V$>mQu6@}xNX&Z~=#y(1sVs=(}5ajP#toFXv}LH3VG6&2CaXjBHoG*c@-&KT2B!bnl!lfI&fRMIAlvE+BGBt{2 z|7RJAPj9Pz`YbL9gs{wC- z==6ZFSh$D_M`1xfWO5bOfkQo*0xlqp1)n6Mb+{^uUdY}aSny*-Pa-kwyAxpsO5v&p zU?IlT*pOb}7!Q5WK{FuGya1k0Bj-D~&6+ zh{zNM(qIA@pV@PuT5oy^XpY4^PO})&Lc&^8{Tp%0J>VWLhP)y9;zxqQEXY$clhqS< zA_yGpO&G;fW+$ZVHgHaeWO;VlG%u}>4l=FvE{nlv^3Z;kfuG|vgXU0%S7oNP#GX*& zyEkMoLi8*PqK=a%U6C1BKQbyrtMtanIs|4{yZZ9dhuYB26Un1Jki*roR@B35t-xYk z>Z=+(xpUlDhePLkN}87m&gl}ZGKc5D+d zxbgJ?hr$F_mwJFx#v*L(29^-u@UvQ4K9%Fi-6^7iQCRpGOGU2_bR57a@W3JlAi{zq zS>Sl43XY{beOP5xj$+gi9={G?y>^CNi;<>4_*|qU|ELPhe*GulVIhrYMqD5^Npk_6 zo*V@VsQkUgFKP90xp1-&RL>Gq;DOq>Xdzpy4cj#J6A&I|@rk+YX^u)0;7Me_xL`7o ztp*1qC@N5|6NXz(KLxa~Y2y$zfJqed;}eoehw0>&N3JbT;#+bH^Oc2S=Xo+^oq!)! zyn-vM%mh2mV9EnPos@%N88JUQlm_)PX3 zoedO$6mFFhu9^N-EQgChnSx*WScUx*8)4dvQa#t;Vo@nN`?N9DIQat_(7oQ;)#^!z zvEp<=S!ZL)96-L`W3i$oMV(L}Z{QBs9QmAToS}x?*96j)OWLj`4iGUTeHM@3ReT~; zZofOK&m@_BL)vh$pgaWP5oQ`IJ}zvx168?gU|k2&@IBxI^(pN?B!{A;V;utNy(sAM zV&F%4WQGfprw5bb1)_alr|}J>)WJ6Y z7FtSmwSOTs1=4@hido<=iyOplv;s^o z@k@cAzy*}ew4(RyS-ry+;xo{5UdU&G4q5;Zd%QtdjT#h!em{c3C&ffqvAZ*f)!S-2 z`C?i8qra9K)nuD+T=1|ES!%6}6Cww>5ExOV+#0kphp1WrF0es^)Z>})fG+p=hcP1R z1bC6Afx|Yp2RAEV(HD5l=ZH;MKIn3=Qm;-fXPQ%DeN^lk`nzuQ7!Lto2MMM*S-gk| zo&NnDb}~dDjwr@hKN&5EoqONEN9?M%-qj%cYS%6i8n&VPT3(!fMryU>)CX(usqj-3 zsfxw@fyu1B$vlEw-s$CEc@B$md#T7HOhi6oK+L~mXC|T8OZ=<+ndJ<-bLOW6kEQ)) zFgaKSrEfgXhj{z}Og>p&#BGI515R_T{Tu|^RM8BE-~fnBqTBxV)GKhYNft~Q2vxJT ziL4VF2IRPk4vt@gVQw|(5dC?yj`w`x>YS$f(>o1Z`|C;)K-dcun=6C=o zOTBcO>ua$9I{wg3g&m})hMJv0(OHm%Cd{i*;MLR)WvqDed09OQ)VB$9e+J39jaryN z`C?)Dk=ri~?XVQFuP!ntsa4^vvPT|?Fll_#aXxe~DoupsDvbf)bFJ;(2eVW3yn1CC z(FZX@rv|vUlo{2k;YViACDyL`5QDLyDP}o?exY9Coy@>&J|agG-*+A@dq@XQu{h@> z6YOfm-`-K<9kE%E3lYe@pY$!F9XdB*Eb8oDlt&a}5u7NbFI8%pBkh%PQOp|LLKNFO z5L=L`slo;w$BK#F?bTv5#03KfP!YrOa32eVDK~PEf>88wkmgrq@8=&vJ4i52m;TD{3t?SnCBZ~+08?SylYd@fjl+T3{#b+zZ{ zAg?d`9!S9}>g(JkGE9kFx;X&PP zx|n8HZ38r6<2T9=~*zVv@HOKQIMFMS; z9I#7wdJL!|w%wwC@ctwb3W5lq%%PjA6mWiP5(Dqg7{ zD&`N<<-!Df*n%|hMhttec6f=qjT^_sV>H+TQuq_47pQ%AbTI2kq6fCfxo%A>i|8On z=VDF(QlY)R7Kgy&*s+s18FD#dGg#IPp?7SI%&0CFF~Kt*U1$p!eztV*Z6Jk zi}((K>&XHjhk;4qrE9DsgI_$mZslG#;CRRj*vapWK6eR5CBk?p^uItBfZ0!NB(KAm z<(EC5!p-tk7u#?Desnwc9ta0SC~&t;zUq?UQ*XTiX_ALbc5g!o*r^lr93cWRx_529 z1aREx6ECKb1&yjPXpwWFZaf|VVTq(PKQWcR< zH1^sH*kP~UV)n*ZqlqkBf-p>xxpL$fWK=a1|YmY=P)j>EhgrVZ8<)sh9~l z;#t@4qxrMj%-b2RH_LKDBg@EDCgq!$$|a!bC=%KOMpG-#1IEc-@MUkQ{Rf~VoVYDc zYFE10jvqMn6tNyJ+D1E2!^QXsefUSQ2Y%>_DCnaX%ly`nDa?Aq{ynn^%oa6IO`M=b zU^)vR!3+?Xwa^L3K0k#mY0-lmA3VuPyHBYg8e8Nruxqlm0UT}pdP^`B$}5xBC$^?Y z$yA3MEz~~1P+`+V6qpBqh^SEunQk_=UD%Fc7%{o^L-uG5FMRn|S!9Ywn!wZAPF8#f zv9}jcW+8_FqpoW&#v4)8PB2004XS7Vo9iNw!n!2i3neK8;Z|WVA;^Ln^}v7>z!xjv z_nDi~;Jn)(Z-OF4x83k!5B{U-ZKzS_@e)!e1;vJ&ewTa{CV~~JsJMLf;J*XAu0G#wG_qbhthib z&pu*^@f8f{4(_3ot0aaOJbDHtnn6adCs-z?d%x2B{mWSFtYTcs9|rM323$Xy1xfL` z9(V$Ku~j>c0!!w@ZSC}?TtqN^ekPD|0g@*n6D2R*?~FZdmreE~rmkY-0`IRgtsOe@ zz?56bn!{)A5L95w$$XPZZFu$W`!7L04kL}v+}&+ zs!ZCQ`fsWbv!NX#gZ1~1w8WZAC*IaMS(`yd9fN}^#=!Szl(>V_zi*Gq=^rnim3o@o z>)t#KtCZ+mpCDA|PF+^;#h*A20c}pfWeu4u2SN~&|NeAtsBuh{&I6&wg{BvjN%7&- z2zGJQF=GzQDRfA(=*|P~M@*!j4y3*NK^$D-a)e1t9h%NyP#w6>mgRg?AI{>Hn3*d) zQ^t7~+w|B_sxkWSM&B6x8b4YB=`lsTpKDUgkYF%f$BX36Kw1?D6(pp|E758cx5M06 zz=E3~N2R?VW~!GPXz^JrEhY*AmQddw4tPiv19={fuHg_2c}i%=Np{g}pv1&Qh)v)# z!A&Cn-Qvrq{bPg-4pIr{RIm-QKbHrU0%;Y1(SnJ5q*?Hp0E;l+jH_jRPIzjTPssfp-FMsj`ZdFwjsSwV- zX%hNtLdLeIxUdRv+wwLSZM5fU{jVb%M*DVMGm;s2pFep1e`XG>e5 zCbXomDe97I;j4TBY=Z1e0GYv!+tiS-cPv60OYZIt+HnbEqQooIo`(d=)!j|n_I*XG@Fb zqopfMCynUgO%gpU*bZBHeIWxw;YZmxa$)Koh@Nv~0CEoNBw@v3dl{1{dy!IRd>}rk@VE;DkopseQv_Hlnbn10 zeG||6UK9)vcKYo-4+0SZ=zk63fJpfZnH;p$`kn9&5MY3mN!pI$I*)^Ec_#%>$7(dD zi;mf5gzK8(pz$>TBt$qt;lnOE5da9vg(^~TY;_B>ih#qBED5Xa-?RIx>Tt0Wp6kj! zQt;kICR0G#Vx8gOeBu?y;^^kry(=>wx2wY9UF5IPI!hjy=&yT4>r&1v8F!gL7diCZ z&P|&+XAXi6m4@qI@n!$gV@N9&7loGkuoot}ehEP01joMaoM|0>9xdQ+y-(NYZsLP0 z_fQ=^V+y60wl`y4EkuijaWia?I{8DLzQ!pDD4!B(s@&wE>qmNXmdJP}NZ=f=ZGX1z z?%0c=W&D0Jg=#&R4|?P$_$@PD$=9=U?YFbdQcW|J@kGmYb&(NA*z>^5JId`SN%GtkVQ)X4ZhD zn}|r!JPUoNgq_<(Z6-AFp^VTAvOQM<&{HwKI#d?6iI3Rn#YQQxtTY|?Q#w={n&S0f zyO)5#5qvyMC~Og;g9y@^=VOo*5OQh=lrT!Is<;O_Vw?*0JVk?AF~D?lDhR}@(RYDcRZ^&N;^Gp({(Wh@y#Id+qa!-eq?_Y)SVRJ>=gr`gkkw-;5WvO3Tx>>%P-P8?4EtJ>6vjRA?Fd8^)Nz0{GLL|?F^bMsOa6N;j!^{bC7OB zDUC8~!*mqGwSBS{=pUZ<<$O7}bmi#S*de8tSC1iku3qffR66zg&Fa>Wt7Je=X;|c% z1>tk#%P&PQPyAduYIr@nW#P%~8-Z>E-TMgZ#Zi>d;hq@D@p_Amo0p$n-laVM_Ta&+ z!jheNq8uhnHf@i~li_RmmG|CKR%hHhL|@-U(swlJGSkjwcUkZ_xe<^0MIF*k?>&E9 zpV*H5F`UO7MOHnJ9s*EkPEWAA-c#lG(8Co9|KtN{loq3 zCYCKI-_6^b2X9~re=kreIk*0E!T}OWRs2pXGf^oOf!LH%A>o9%Ug)@+qq(V{`KuF#Ha-9@>nf>#L~D3J5!z&rIsfe8&6%;0ym-LP zwa@0?TS0Xb?O%0+zu&5Bo|qh)&|BFRI($B^Y(l)l;RbIkqMu(HZ&g%wYMnX?!;Z%) zn!DFc#&=8}=%+h&INT1INLZOnPP9BI?c|{^P+qb095*j)l|y1~r+1GqS4Om;hS`U1 z8|OR6tvI-uUB#r*W6j30zfa~ur&#zYHff4u=HmPBg(H4WN~GHl-G?W9n~oR{ct_n< zIqCBHu4TE|E=cI@@@{9J;oC{1tJ%+QA0v(DIZoI4P1i?Flm7@B{@kwYcg_i&uB=OR z_C9HNvg5YO3zu)Tu1Ee%6=0?t*Sby*?clg~o6LqIE?atYJx3Y6C8|zcv!!D&z zw^J`}UpLACsFH_&if~ma)AD{RiQET5D z?m=Y0(7pdh)Ox-De?+bOs>K}&x+DKv)S5JQNo-WMVp;9vbAFzf>h6;t>vIuhdix&? z4}JR|QR~g?=}YKK|0Qa@@$-vttW2c3XJq)-e~ViG8M*oI&-MTRMXhht=>H{ZjSGNX zH^(wC*WZ&@xb-yfIQ5xQJP~#S_kN|e05@99iu$uWz3V>p+?$5b$;VVkQo7D+Q_aFV z6`iQgb6R1Ze$XL)W{={Z?3+E^T|@cf7Kmi|-+QegKCWGvq3eC>e;lwaYMqa{@afu%jE|oN zgoS#a2Ui*rJ`cU?yzu$@+O?0LZ>&%1eHs4#G~vt5t?76At$9^(l^`wlHKFfWjk!4%UTZT(sZ6Zt= zSKHy;`>Ro>d_O;|Z#$iDEiNiw@_*lbNPdE<`KLX@p|x!O+O6gRmHOkwy%+!e!oQH7 zy*@5?@J~e$9}gr{zQ$X7r`G?}EKBpM`*UY3&oBEBetm;$&+6aGa=58;QK9K}wllg} ze4pg(CRx04R{Vmm%+EuXp-+&`4&_xwe@t7ch|k*GfwgEBI9%0@<%ditU z(iF2Vsgk|31f@|cPAFDA5Uu?03nq3x$4kdIVoLLGE0x<#)MW6GPXHIohrM1>KTf#* z4Qj$E?(8V|M~pEig0_{~8svj7B=94H=10pVORGXx<$)??mlo!D1Yzp7cpO7gD@5aubT56e8($N-NNKyioC1v6AJi zK^2)NSbrzUm>u)wxbMDfYZEBvdy2?i zah@0Uw)Z`}di}>mX2pfjw+Bx17JgjHsE?HU={7eafxOJRxD@v5!`$d^?Vb?j#9&lv zz}Pc%_qe^jpUjP*+k4&m*k|;kUp|`uat?MS%RiBJ+5b_=y*kg2?-M;T^9R`h0T-Gk zF2p#6$c&4kq_i%dsfem_dEOWW7<>AB%1J#@^Kdgw&G!1ml-exAMX9-iGTz9DPYYh1 zjwaRv6Q9#zAEt`UWX`&`Kg;lqe^^Zo&}XNU!qt4NRMqI9)_*^d6jPsnRgg-`wKj^@ z#kxLC?z_cNUdu@juKE4p2kCjiyF>o9jDL2UT4NYMZtD$KlMLzO(#I+a_DO}ma+9k& z?-coP*cuN{W1yMBF_oSf_F8BlGL{cfn(tXiyd+nHSX_}80?t1I8r8J7>6HJhgT;R~ zjDUy0N=#~4#w#m>Ol6w<3ntu>?Arp?e~ay9Ar#2M#+*bd{9TN$Z^d!1U*+)*GS^)Z zxfexX+$vb%Qupdjv^XMm(BAj-aCNWp_;@Ti&!ECXSc4UaAGP_YqxUfAym~A;njtuq zyH@SxL)C7iKcK}l&-{Ydi`@#pd`#H{X5m*;oAdbr>N6h-^Vp%WAcK5tXz!VSV8?;h zH7fbXLVrwyIo#OF1ywx5)@pn-?%OmfbBP7hXVQ>Ke{bW3gTaQCJG3wifr?=-!WPL9 zoA}IcddWm>w+?aMr0xn|PprDq+k@4K(4ed%QTa;wFeY|f3u}CL22D`c%E%+;GZGVg z6%77}KSK>N;j~b7_|=a9Zb@seqM#KQ)|q_iuvej#<8`joTh7!pvp~7{hJn6la8&lo zG)HHg_b6e+494W=M?Py($MD!g1M}QHpMMb1O`{qwupz_8R^UOz%ithJiOtqK;S>G- z0Qnk_mF*H7M7C-Wi-ju9pFE{c%~u=!w7e}FdJ{{`!)6*{1EtxR#6j>?87@exu5~r} zw+Sw;=X03ohQ_vB`lmNs{dx9JZ6IzC|CGK0N#QI}c*sc4( zeNOL|SfTlL=eNikDR?rCQ;;*H!9k_-uOSUPQkRIxcVqwlcDj3BloLFIbnM?cIMRvC z759iMT2s)ja6UgHF$$;6SRaTZ*nFkAPJRsYzB$E)kIW%Y&LFcgw(GCxkpUKGJyo6w zYXp+2$AL%A(5bW0A&nWHGU=UH_m$Ea9|O}4pE)YED9d5u5=ekO1?>b&H=c9pblm^y z8W>OVdOHE=HJIP?^Fa4NswkQD+Blq6S|cA(K|yCS!PP4VRtl3(vXD0Xe2x%a$b#+5 zhm+FSSAJx|a2f884_z$6X%dm;>%MDn8P{*FKm4S)(MMEM;16OW=(2^dS&%&g*+@iI z^q?vheH>Xug)C%C55kOls856G%O^dXgq2?!=nH2Q&m9#1*N}C60Gmu{$K$<`ilX^I92FWBvyxmL<$=qYiM5S#1 zBY0VLojaNlG5nB@Fpi1yD;t=sI3~)YYjt9~x~fh+by|XJRUq?j%mQZHnq96yFU0{7uQ)Wx(eQDVz+I`e5d6yNUlxl-4{7o8<=+Jkrk9I zZ-%OmF*y!$prET}K{bU+h7C&32FnHme5rD<#&yO0tMC@)u4fr22Ocn3l#gb@D%S0{ zb+Y^6@I%W7z}EtAIg!7-D;utjobq=4Oy0k3RF&reRx_wrY%PnbkybX<9rTgT<3T!={LE%cXL_@%c3oF|-%>=)Q>XnI4NDd=ZtoH@Q56Oa*8dzo zeAcq+9zmPf5N4mg@8&$iUvyll{J5tc*0l#oHv3hcH0!j#ud%qb&BFmcssjgCcA@=Fg_cc`!2hNinz(H^cFJF@8Fv2(F?Q4AY#+DNH4huob%!r~DCc>k_k4sSm4$YJeL9lG@q*u<*4f^>a8HYnIE<Z^ z{o|h<%?z949Gml;H^-4`C+9gIny5w+!aXGs)i%SNax5d2bCTqg(Ba-33aNCK)IFjk zl@7OC<=cMy{RP_|kN5Vu_Ih8h=hLi9PW7iUU6SZ~J^A)^IpYcAtuo%n?NQ8H(V(&nlfp z6i*|FWqO^ah%7QFtdSG6wm+>vPPHbV6nHU&(_G!v{~kiXDd(%XN`37e{nAR2T6$a;EsW3_{VFiR$CmcPfgY0XKSIBzBQlSAsbC)oeKvz%*vPXY0p~ z>gg`VAduiG>rRr{hf?pM*nrc_o{*UW?bhmiriR z<{)HD;>A{ENnjI`E&rK@XHLs+S71W8acG@l)>?nsf67D&Ty;>sgpkQ9gk*@aWHx41 zOEP01zmyHXx-WNc#g*H4$_cdNeBV zaPlKSPXsMkl~?>7shv5vcG|_NOKckoPGXf2yCAfc-mEcz#g=E>+3WHj;wT$)aROXX z*7XSLS4e^}%EXmBy`(H*R@q%Cr`2HwxKP|gBtw<1x3d<`FxhewXOkRfft$at5OH^T zfoF+bh$0bmPqw@;vOh|4&5{k_)VFpj!2Pr#n;+$)$Ro8T&33-J< zQ38KF>p|pHb^PnC@}3`aFI%nxU8u;u%bc!rh!jueLC!osTRT`)f==M;*1R-8RIt&VSt%UD z$aC=Oak25D1_ZhWlFmKOotjCNI4H>xM%bSBP0PDRw%bs z1Wv6-?w6c8G}pA1EC(CERk&1(lE_#Xttn_2v&&TtfT!p}Qg3$dr2BfGW^*_cEE+m%3G1}C)Z;@b6rwEQf&POr*Mf>`|O{@ zN9QU*pb3~28gdjT+Uo%jWiy#o?@lt@g#B7({+-HrBJAKJrYci-h!1y+HIF7bum?$G z2-zB8O9y4mA?RJ;AnSWt^;aB*I^vO@N8}Ly)Ph3T@?*@3i!RW>!uY~%Z+!C~#)RXy zcHVyJz+~Y6dI%Rqgm8d47-okxqI=VAqM8IvI{Dr5w2< zH_t7!0wiRp$`cb6I z)%WjiwQ0llHqJmlo{Bc^)XQ{O@YH zVN5^bwh=jtfILcu#H70`l6&_7dy#~PDI)W%%^P3Z@*Q`9^U*JJd+t`Spo|!Jb0GZh zv6mHW-Dq?9d`VLbe2(%4lqGUv{hCg?ELdLO@8nfi5DI>EXTA6Xyl&d%MTd~X%d)zf z(5Y2j|9dt)v(`}5B$Z5Bah`IdqVG(yTpGG*t7Is0`Ec30Pg>Jo)%0P_oWUc-5EZ4p zDFqGI_n}Q}1i$Ab{(s*8y>5Q1jMmcLG{Vx#&cfxlB3z$RD0#H}6oQR$)40NBJE?ZyGSC*CgUx_;dku7BN>1vz21;I5f+@Py3xU`}6D7RHAhqQI zek)gW5_OdS<3}l3E0mek8PCZLRx5$VL=E=_i)VcIr+;n^B*qk+OuipwI9yMI-OS8# zjr|&rhry-mxD|4lLL(NL@p$G@zqrhfhz}OX8b<2gL3o6&9WpQ_XHnLj?pQbyKJ|>b`oX|0-KecgRUg!wgQ8!q0A!ICja#8|h;4!- z*YT#M&Xh%}XCY~*!I*bD=TiA=mBNN{nopY-E+0O`hk-jnasK((<;UAiPi3vDv`l93 z;q85iMY$din10;BlzPDS9G)dE+wreLunBU(jcanj4xcojgcS{|**Fv`7umH+yvu@G zWgaoTijfiVNt+sK_x;K7=eI#wiKB{cFrVcq6}&lS-0STxn;fjhp#5Wb{QuJ3VC=4t z$GPpX=OB>o2?2YzSGxDbvY534$FKZaD6dZ+9& z*dLqf7VCx-WYFI`zHBZX8%D%%D-0jb3BoQk?CZj~iC!fPtWCZ__1amD$odGb@y&No zi3-Umw?=_{QI^Z+OEmAI$t>I{p#~lqFq!e6%9jV<76X1Se+Ei}(y`+=mFD6XgY?wv z1MlKVptlSI%c4gu6#Mu?BVb#KUX)Sy`1%DEk>x&B`>Z41FAnp-zJ+MN#8D6PM2E{D zWd6p*l6KiH!wP(vWV~FO^P(!(w{S7(aCH{%oQFN+o$-!^yb<52bFGMSGuW?%k%p6V zh*Yz}4tlVGg0>kzDHnZuXCR}aM}!_y8FfUU_y*O-)2a;S)Y0hT$w2A8GDCz70FQ8M z(1CS^Y`5uRAxhSuN-{Gc?Llm$Hf8&M{%zo0Kh7K+DM~@#saN|ngLyX8HGJ_ z+@}5p-`Mz(pvZxmX10qSq5oKwt9CG+_{%$foq%Lm$6!jZWB*k+6N<@$ zevdLsq11xSMBkL{hPy$2WqdQGZh>!LqFO5HbkZ;^A1Nz&o@G(}7XnQ{-34x(q$1OQ z?Y9^t8SXIqHIrXY>pa~RZ;~-EnjG6(ZI?k+IZ_3+(3D0u6-O{!CMk#vyRwGRKRK3K zq?ZOBL3ceEPXg}$^2`s1=-X}Xvxvf%gBkD9R#x|2p4AN7Pte*gOxdzZNytV(t4}ax z%`m~|;9_GChFyHn0RoI^$FcSn8t$+nLCy733)qe^$oikAW>GV$&aU+1D^Heo7*pVc z+rFT!UQfu(U!~ZthT>|;yX=^%?z*r*kjDVMZ!vtz@!xk_d}AN&$b+~2$>=fUF`=ML zs1pwCsAQ?7N)h403fIOJp+NwKB(tQV#t{|d%Plq_L}fIky^d1gLt3B~c1X~;qSqI& zvBXChmcrp=4x+935*DiNZ~~%Qt0qMXuh9K5P)E?_4opGa>g&hjc=upmPz!bhi?Si8oQhz~lw!mgSp&MJCvpE)V! zgZik@DUbh@br;vG3f=hTdQbB_L?RbAU%?(_JpRcTx$du@`wU>g@Zjd}r%EzL{SE~S zpRLI$ln4|&W7hnhk{Gq=p~hwz%=&hX@oWyqvUGMD2bR>UZBzByUPG3lTP}g(B$0r& zh(VfE?mH@;HbMVThA-Ev9;i1A#>!v+04)T|tWmgCynI-n!PV;eoP;eS1y1A{#rMsj zM(EUnZGG;knzu(u^3%|OX{p!_?HUS~1!4+=-+P%;k4@NwvOl^<6V==O=LUfAHTpdI zE3rw3(@ZjK)!L>vFs$Fkm#c9wL}$!!cb_^Swx3?A{p^~;3W;CxXcUKX>x>8{4f%QjHu;A8^5}5W}nwoL+J^#8|6@ozatkHNl z`k=n?TU4}a$VP%9<}slCa(wrV?D4zcpRKvE9#8JPW6)98dGWmYQpih+7Qpo)!9}p8 zNaxm6cJoHCL`j-AKCbtfulK*viT#S}qlyc`sz~A4?HgH1Jm4j%sYAB+H#0)Z6{)~v zjY*dHklGnN&Pg^EU2FhWjB+ZIO;AK6np=AG=K-gh(+GDrWkK!XcPBf#FhMK|e6juX zG4~_1q#!_@^hozKbP2zhiPUAkG&o;E@UJM~8Yf1nq9X!QCxZbM*9JAw+TckV1XXpW zjT3JVT6E7|FcAT%cTJ*H7!NNBfNq&-8YzcQ@Kuevj%$dxM4oFxgxvnLwfn}y3cGK6 zQQ;O5&!h6LqfcQ@uc>K!WPb_X&wNs~mEJ!f0Wi{BAL*VSPcFod`(F4hr&`E3ySZxT z=9?+~h>k%+ic(?DSz+N+VFASa=ZBvEedYiA;O4;1sv~L$iZaF~RiHz+->R4P$G#@VlNlisLb~9W( z5|=R8yWyj<&8`&5;#j@Zobb^s80PjqR)+ml>JHq#CR8D$mNkp8+AJuk-oe$a()DX^ z1=~cZVAGe?`b8-0UMTQuHfJ6y{E>=!5&_zruT`HwYl5p<4J|*06!twXS0W&9&0^up z&5wDcJ-_5^+6w<$QTKh7lMB@BZfNZOUGO{_ypAt@!s05Z$>&b!bpnd?f*RlPZJijE zf=KksG8I>V@0p22Ov6%w z>o?RQ2Ycg-grk_U_nZ!wx#Ge3)8_^tAy_ctFk)REFj|I6GmE+|u#_n{)aFa)EyO!S z6x9NoJBmO1O>}kJH~JY;hk`-*D511u8G&Si`pG%R4wXGOke4gaS|#h>ZZrPhGN3U7 zkv!7n#}|P!&to39+?vJ^GWeAirAsPMGGJ`eW%s{j{r-cP9*Z1*k{W!XOfY!LP#)(R zza6pFhIn5k`h$0)42z7-$4PtNj#V zOSRi}Va5>sQ{%lrt>}@u9EHAflCxFDCS>QFk3b|@-8$b@L?YPdvJl=MhT6*AeGJU% zy6et0*J4oACm|ej$(4&F<=@eF*AMQvKU?qjc^uiCnsw&klJ5w6k!Ctp-3<`r(+s1HgjGvf}3dAJVEXqXzO@o3?NmV zYje(S&>JUs+t1H%KSbVv134`&3dcjs)r}O6KDD%3!&-?hc?g8}zN77}Ml1z66$2nL ze(1;awljDX9F3(*TzZhm+z5qd*s~KB2s#IHnL}_LXoYU=Om`TBgchSF1YdBxpyhiH zE&=NVenYB~Zv3N*XvC8mM5lq06pr<_8aUm}Y4Oqmdsr5bSn7otjh+aDDr#~5#?4xi zs>8AXiBugh&-veIWzUH3|9}xJyZ>94P&KgJ-d<0dP<+vWwmY`Gp5qilldc_P4m z68nfq1ti4zo}Q3@%gagJruzu;!17Q}Gm_5sK#D-gLFJnX&~p)mo{}8&#eX&4?1a%{U998%fag8EU+ivKpO1-(FxxA-&NAX1Q=XTQxc5p#g zSL7@eN#^@+IQw_I*vtosp^Uggw}n%<$};HcV{aU|_;ON^cV?$;&OP*f6j*@C$|~F@ z;Q=K1KGo7V@8`v*FQ0c&tvt&soxZcfrKk7MDV;zUm5Te3#Lzjh-@ss*Qkhcz#tv>- zMbS87D!BY{gRnRP`ZybDcc|gdjfu!lI(AQKG=cV0L;1)MWa+0TC>+lKjp*f#U$x_* z$8;}>eWis2UhhzI{S4&T%hUZEa(RyDlp%d@qk-80)$E`_AqD&~mHDyS;mRhDP$+`92(`Fe8o$p$Uzu1q<3F72nnOt*@ z&S|zL2dr|A1sr~fem)&wh3D2a^4$|^pGSx%{v~P*7%v_x$=gi`&3vJy5&+L9}TeLRP1xn2`pl)mzjrg!b z;vDT&Kw{a>Y#eDg#<#>@CMs}#?g~6?ID`kFJtVqI30T`a$AJ9y#u0_-9lU*8bPqX%{{C&TG ztFrd$k6MK8_(&joitMF!iNuc#P&xKJ{#roX#xsiIHjZFeB( z8P5vxDuwKyeb6eX;mksBp@Mm&dA|hmbmPgljHUe_IVx*gRxG(&S?Yw?*0g`CKI6bo z9+lbcZ?$C4vlBwjkCz3+aewl1f?}gYU5?^ww{#F3%QiD9~r5Lz{pGt zm`vU}?BH5o@gUV|#j{;fQm4GBT0_5&}hYFRu1c;SEfsvHP5qWqWOYUC>4@BhW_ zx&9>6o?msoGyI?#=(gm_jTJ!GOX)nA9}$9?E^jp-Pzu|qRA;4JU_FjiC{u)$PB}IC zkB1DDm29TsbLSM!cMaK?mJ((9Oo9>m{3&TatMsRK?qS!QEr!a58{IvhPuN#3+x#eO&^)JA@~rq!2pF^I#o6?6TtOJCN=aWB=~0%88xTG z=2*x%Q0brFnkv~)XQx8elKhhoG5!xYZXt)u7fSwGxYWnOhh?a*vrqd69;U>jG&1+w zTWq-!9({p*S`h^KJ$70>8$k;@H7u@;={7pA4cq!2YJ)((9D+LE&Y2yCgoq9Y)I-at zW$#s$wKx8Cd&%0hm>cc3A z^D8}v10?*atn)qX>Pm3cuRAD>ouq@*Lx5(i|I0ejn>5(kq3CM{pI!F$5~V? z?s>r=*YKoF4{aPiIAr5=a7iTpyC(-yOV+o>XIhF5a&e%o8^Up17k|20Qcz@{QT{vy zYE_>@ucjO&5(Cztk0SppI9YQnn3&1#JF{3xx~&`H~OVgG|)kZ-GsJf)hRY# z02Nlj?j0stL2dc57T9ZVoxOJ zg_^$d6c!^1HB`{FD7_CA_%?aS&r&E9IEn3wuNMv9Z}Eut2^?Sha-rUzfsbmIJbF8J z3P-E3O0bK~uaatSeq?<$_BtiqzGrnUK>T;L?ZKYqK(Xmmjl)2@HQhYsz3{eYu*`1rXe zgy*deo_TY#{mniVe%B2HBBM4xpD_;*ZhlZ*_Ij?R_VN)wWSA{TcWO_M(fyEuDf^wTKT`$|h+?Xdty`u<=mMvF?DZdkZ61c~<=-E(;O z118W+QXeL727A|3CdEGsi+=R!DWK6r=T0-Wb)pZ-q2m9OoQ&w^>+Az80Jz!T@BTeg z6?7~Ntk|9Ij4P`(mH#J1jc8Wsw{5@IQd4{T!N-}VK{sq}er-SaBKEVtdHYak=c~^{ zY89j8b6+1Te5v|2`P)RPr0(mSuP{2VGkr$Ot=s#@E|kug))$4>LqWq@Rd)=E7~Qvx z_C{Sz)X$f$6UUIFk~>P@_jB@bCQq}h9!%}u-0xi-7IE&_{x?4ksdZ6er$zdpqtnQ2 z2i{`L)&2i!r86*IGK7Z05LhoQX)uuybSCQY*yr}=|BQ#lxuwsp!K3D1d^?oyIC;ms zUaQ(fo~a`?i%{bzl~i$`gbt6TCA^YE@L{@El2P#e*`aF|OV4_eRUod2Z_Vv*_nap! zHxzAm^@bdM^Ym3*K9gzeOdE|pB=kFv-oRX zY~Wb=P+%)WGcCJOj6_pGeSW^}%;a5NJVV1u@=1KBA2e`yfhYX;H+eY-hb>(j_ zej)#PT9&c4L=PGTD_Cxfm?W<5W@H(u=Rb@*yb^MLUF{`wZ`;j; z!Y}97z=3yHCLV*2da|(S(1pW4*7< zW1B^AW&6_>yK9%U6jt2xS9~u9UuzG)Wf9)n{_Khqx;^OhisPWg^X~IijeiU+{mcuc z;Z3|s5d8=`YRxe?amb;pi@!Af*{9~k?$PMu=RVpx*PeSXR_O@QTivC4nYlFPNaMjG zUOku{3@BIYDm&exd~c^}C^)(Me-$0`r!ErN;or#*qa*69e6?QxbUbzZ7SiM73UT_F zamUGIYgc!b$Y<@A0Y7=n7`6Bpv#wKD%CjpER9>FDIxXL~Adku4lP~H!vR-%Xo4l^= z!LGYJcG^DqPXp@_yU`ZJ>)%4Qp~#A1Pu)cEeRd1i|@`{yfq^Fg|qXsqplro(RXCz9m1C zUpfCLK{CgO(W}wR&V!8=<=PQxiD9RIrOX#bM#cQai?w0uf9Rg==PNXobhDk`zLV4hRZCD)_<$E#QGewV1q(awB(v%Jj7y_#j!@51*~xfm~}E#xpXD#xQE@a zuf;z@d5dO^+ceLrOW?`47*SGOdbdDE1zBv=-nl^>ZP2s@uJciO?0vPDK4{huB+yb8 z5AfJUdicovSq8IrU!gmQ>5wB|75AD!$5&oL{qNdD z0aA7l20z&XlH0W#gi+YK_LUSBNpWb>y1tfcE9%nx59eETLke~MYu>V_YVC}2Do)qw z!wD5wJw>Oz-Ys)bsQEDu&$?=f$fkLezuMh4#EF-D!cs&<40gTE%jXakwo2vPacGl1 zXno_}-?MoKR`W8`5sBTYxawn-phoB^S8@Yyo#jYv0RWQQ?X zfyIfwDKbjpsiNzeMuUn?bP>CrI!x?5SzccY%JsSF4s>yy5Y%@5M1V|$DiJ%DiT5<) zj`i*JlRO_u)==CSCe86K|D=xujdkNi>ImtG78F3%`X#I1Q@qdFPq5My(QY!FCT`5s?#>c_*st5pBD$-|!ETR1@8Pbq6g#n8ERYCriV&p*5);?&7|f>ZPLgsi+%VRpGQI=6pohCapPW)_K3m&A{`K^MImS z0}d0MH87h^6d4)QPIKL1Psz;_wfhNx3Tni{jiU_x2hQNQ_`yI)AlrB_lpe{87>jkg z(9QM^!;%%6hYdPjWd{Oqwv=PEMpf;I{GIzO0539JqoUO~d@T`llxiRmda=i|Daz# zN0!^HC)5Ef-J%P06{f^Hc~dU1OSFsE@)^X~9e(Gc`-pAVezQ=>mQmgIFt`GmdvTEw zwBRl9llt!-8khikC}Gsv0n#p#+h`b-^{85#{=P1e?FR4awNX5)F@tbND;DzN z{-_q6CPU#J4}0K=2Q(ZYRvlW8Uc&!3vy|olO^HIb$cxg}=}-Tc1L)F?@?Hc?i9Ig!G!JNX!MTT`*GPz{!BupAk3-Ovn~P+BouFGu-f8aFUon8ivzFI~Zi}T>O@J za;?9x=e`_#iY$yF_aAu`$^bkQdGOIYJJ>QRBN>XYkdv6_)`)gY0diGa;Km-g^6g00 zTSCU7)LV$05}}gFU}H{`2h;n!F~DF#7$W$x*iMZKY8Kna^Dgef2;S%F%PQ_HC*c_Yy0UM|z zI?x<(Bl=NDzg2PAAK~QGes=;zUDg=wyL^M^&O9OI=eP3+H<$38EjpnPeT2xJ#U-?X zY114+e2!i?%>g^2CE<6ka*0up%v!LL+J!yA4JfdO5y8@IkNCy4QIpUdCkKc=c6i4C zhDM;=-A(I0LonAY8C=^?np0Gc6-sw+FmNlJ z6Ba(=2(0ee{HSGh&~a|~pW{&NjbTRz{n2B>y##DxyD1CLu^M=%kp9Y#L76#r*@{4s z8{Z|@#_P7T(r*Xv<4~LisAUo^fhG1tQ0-!AP7Wh@i9BsU>^M*#jS|}msr!r>c2U+u z2eA7TUEZqcA5(}vy%;oAY`-40X-I!*s} zN^hgCt}$fGfav*t2+`{eg#|JQwzLY>lA#hl{i0qA1`+8Hb2irUO_AT`u+}Dc@(&oJ z?hBh6wyxtCIMAl|fce_g>rOdDH)>okU>;9%=9oRNy==Y-R+MlbhvzK(f?=Wz|3lKE z)?wI9aHh?t#Z2uHDpcQT#OO!9APu}4K4QBjPAnWZ5~bL;P49>ow-H4Oj|8&2^ z6ZpJS *#8M2NXO?Izo_ubr{%t!)8^ngN;+7&tz93*dfvfHS$%fYQ07MY*;BDll&P9S1gfPX0 zvpKo2X74{EZL?$WDUi)F1>`2K0h5I*AGCzS`ZFt63eR`ueIs9jDUMT}(!g8jIRrPb zp;`)n7a67HSVe&p2U4b}V_H!o)@*K*7VWPyK5|0Y83C&@Kvt;li??J-8E${f%a&=8 z3_ohC265ckdk0=1o+sPaotG`p@4^GR#UuJNs~1fmR?YFcbHk2lU|lR&mo8nUlwB_F z333t*vmck*W_+Jaki*P@^!r-x%;qTxsrG?dx^v*n81ZWcmxzTJqCvWwU{f$E0k|K(1s$X@Ky{c?J`nr)T?+DYG1Vcv z)5}5jv9B(vaBT;uHsjQ%#|yRnxjJb)r%kZ#npo?RDTE;v+s=U>>had+M6X#~nOg-j zK(RxZv2cd;_z~-R(3@v-|8f2?`NhMbDZ08z2~ndad#H8;V2=wFrW@}=w0KAHu%mF; z6DuBj!_yfB>9awkBO+5YoAO0$R0krQr8>Adyy?fcB!AQzpctwB&>_a@7UvL}uf**g z(MOPdyw=-^pgqpKliR6|ZNq9+Tyq`T{RwUDO)w6{=}ullj`BQHL>8m53iTym76%WE zsIq}`Hp7^$R3$#hhMWOrbeo}QI&KtgAZLd^hwsQWvZCSJ#JbHCG+P9ZCmTTb8Z2_W zXN|u8gk5Xl=Fy6O6l>(~TD;pDZMu&C#%qgZE;i)kNAaHZXnhHih{pj1WJybkidxU z-pmOd0r?M5?Xz|N*G6?jP`{*nZ;?5hu(Z*43p-ZDI|#jUJU_W{06osR`r9>d__Ew7 z*xPelV3p!1qQJU1Ks`4iWS7f59>65ae!m=ztq$LO^v>14yO`(n!vGTBNhDitw2h|O zi+7&t^LtCagqDgw`h!$!U9kWP#-`Yh4>J%VHTuZ$!!xBsk;i+mCVj+ux)p&ESn5Ks z)84=+*9ptZ+PUP2pbf0I1gg>;lc20|9ud`KQ<0Ltji3wF%a1r9)~UXjYaRTM-c11Z z`0nW*VL-vqbPRc!>_vl zt4->O0r9Z&r~!KztqWD(xEeLbe|M-eFb!ZS^apZ!mwtgOF*Le;zh9_(S)X=j6oqnf zY3JjI)@cwc0<@UTlq;h;q8^|O56dlo`|FGuWX>slpZ&9+?hl_F^^H1c76LPjhY&NQ zF=>zFFTNjUtitXpM}@6!J!Is6T?{h(OsOKw7fhr)U)kJYK&Q~QxV6$XdYi%Fxa zh#yI>^HjYr^E?HD2o(w-0ZfkhzzAtYb{q#Eo>d^XJK6*8Al`O}qr4#+vc4ODmci^n zP$uh=E7&ofCl}9=+ekIZZdSGepFSvpOo1Gy#Qz;8gZN@Q<6%g>2(D`(M~Z}C@W{eMPIm{X*x;cnvKx6t*zmto>ozQhAcq>PO;Mg zMq)Hejugo&nbyw56B{USq#ibOZQqJJ?3RN0=G))ZUv`Ld#y(Ks=pqs6E_ z{SN5}jMTpS*L6*XQ`r3o>~FX0w(mDqSXFVYSO#Y^D&g}5rKky0BrK&(aR7;)aH+xn)g;HL?6h9#}&rL`iO+N0TpL@#g~{BwZO8xKoYwsZTM0GdISt8XL3C z+IDL7^VLypTe&>*6G3CPx3o9iH8ilxMIcygL96QajrdVCy;%5AxYwfo6v(h*5Jq;G zEO#G3OCvyM;zfag3rwTkpRm%V10G*dV;kvEcy9@{k+RM=g`OT|khAh9xE_jZ6ui^i z<{m1_-D~&n!3NZ!x8MMfG_^^~YAqhF5#e*gct8v^Nf<+|f;;MNTJfD=7<0`;+o5yL z{(kI<50xN;;-J2qgRVc#u6f`^joQwCqQBV&wp^A}B)(rD)Euy$qv?d~I(UTwn;d|b zzrsu2*4i_3`n5vRTk2hQnUjppT6{#ZJEf}4O`XNRzykrUu(o(pH-#VCX?327p{&BV z=fB^hlh@E>63iw6UIOinVB2UM%UQUsur+lF2G^MOwVlV;)-)$j zGAmK+qZ*aHT#y*#Wrr_5d58?krN$%}8lZa4L#mxnI};k^yIQxhcip|egc7=|v7ZCZ zI_EbQN_Ob~Ts*z^e5Q2Rbw!CzwLt5!g}GXVcJQ(ash zAqaPDtT??f3@``Qa#wG_b-{t%Q3z9CS3)?7|z?gZ@QC+`+-jt|4J>f-foCAoS9biNF9xjYSOJ-^?m4i+FVrJ9$N1j z=FruZW3<49-;vNeemUbJ=Y4kG2c1&NjxN@V`rvbG=$SQCQf<_7mGUBkE+<3TexIA% zm5Q7~C;UsaaWE@tYQsR*%M@wQ!;y10J?x94i*1~8l%F)m)QsH4 zseXOH=1??ZZ~T9yBOju%$2ou`$%*R@O*W$2?*fI2Gy<&NYTcWb^;CO6?P1iE-}gjN zC#eL{X(HL8ig+~>6Vo5sL4{!g3v#vB`u78lrmEdKZ#=#!gB?KFZZ8oO`Ro2K>zw>;o9jw$3k|qV%#H-ONB9*Nu^#Gcq}OKuphJA1Ag+ajiK6N(MYWk= zFaR8A)X5eyk0}k&5PLpsPiv^p%y>HWXk&5*!}arCL~Q-{D)EW=5suEDfoz!uJhH09{V2Hw+#=`Hi&f7%}^E)pt8&U+!;juDOA_88m|{9IrOyY z)$HotmFM6+5<^&Gk?(6eBFgbcSnK7DyO3d#!k(%5w8Mahp#zu;c9?J1=6HW9oV<1l ze+0YF(iifBQ*^S;?aSFe5R-;c&eSag|1@9Z*b1CSW5{$j-4<*YZ|ba?k9+ zuKKIL@IFQ<}jyul-u{#k4H^^}f?GJRzEP^Pl^3(9K;tI#4HWdzvzpF`YGu-`I2dg{6gU$ZRFD(Gv^~Y4Q zSNz~#%fByWWd?%S;~r+<0*L>B0g#8&%H}3#w^Tp=5~_dcKqDvY2R(3+_TU7l4B@V# ztBC5PRMDG53RiuO<$-^6gwB}y2Er6J7ll&j@-!=s#d#ZiqLi|h3rnG3jjg|@)ia8X z`~;`~umH@)Y0sKCP6N?xzW+lkZUY0)8l%;8OP>9pzrfeC&UpVmwj|vFRzAJSZjTQK0Z0*sx&R6kL7uX-?1m2_fkb4qQIy$1oT#mq11Om( z{_#s8NZ>IBs-d_mp=DqdpyL$SMy)<2j-%=1*?wC#0TB<@;xrs6kYE6^E|=QZ2(TVp ziTez~oBf3oC4Vk#-L)Vubt1#s;?$d4mGh#_jWMuLR>Cfhs%DYq51dJJ%1R?GAI?&? z`M5vb>m$!|^$kYKht}E>f@H&RNv&XyhUi>F8Q9 zT6>YPPF7gw+hED);Vw$+stU=Ta6nOcoz#5&g*0Wu|+K(fninRF~?t^=yrK97la2bOU<8$iQG)NbDZHIDKmNJGOAt^`t8ya-{5v1}ZGSi<}&RvwM6NV5f8 zmo)kw$`*SR5TdwPA_R<63LdjP7WdOwJRJ{HbOk96guy~Nbp?|%bJ+B02XC|TOEbiBRcSzg=Dc+SuCeFg(8WvIDaK)mgr zB~)jF*-=5heuLI?MnG(|E>bai?c)~qC65)n(vaA#c~!0=+R>a9JzbjYDnUskc~ZCi zv|^lAEmvWv$wrMLqb~x#)|9nQod`fbz?8*wn6khmNy{(} zgu;OsU=u?Q#Ez?Zn`6Z*XBdzzD=?bxRuIR^y)~n?hFOs8b~M`n6F!N1C(+!(QaVpL z)NH6}QjY6qDX?&+2pqTqtoe>()tC~;u#gEhL0FFRp?SII<-N~K^!KWV?+Xhbd!QfW zd+ZnO*&hlz>JX9*V3;scpkt5#kloE`wu=d=wn}Uka@89x?c~EBNKkW}Z%$m)j0CyJ z_Hb4qf@x)|<_1R)BngMuU2%Ml1KSYQ9m;LQvWS0CcERRRXS5dBHv~((ATegWT?uCVu(T!rYlyO>{NE#bBJXw54N%oBZ zDR#L3A8+>=)KnY(5B{W*ga82&NGKYR-V8l74ZTC?0Z~I05D}0jsH9Lr@4ZMDP%$DP zh$S=uv49;#4JZg$9_(1N`R)F9c4udH_T9eEOfo0u+~=I@y6(@{UV|as$HYQgCB>I9 zav)J~3))*1Ysdg-3_MD=P>UgPYD@x8aSOX?o?99=cnm(L@#QY%TNeY)Zy^ZVCgss} z@kIHKY{Y<)IHvO9(5hsgHU2vT7WKr`J5u7N($9|QT)mKR@o(Bl91n+x7xP6E*N_4f0Co$- z#qvdsLU1`DIFs?d&{mmW{Pp`Rh+H3%KMo85@?vzbhC0rS2+<&f69- zH##_x);G=*fqvY1f(TY7BD|HPy(ztKLLE&Rcptu}x;pOMO;j^k1bX+FS_^cDqCHNB zuhDUg5TcO{w%S^{nO_(dB1z%_L4Z#KU(!$s{j>$SnWNx32$7)xU+2SZFZf-K^WPO% zu3lGe+75|pWuLNV|C+$ctnS2qt>_KGW`**)%*(1qLhtPL3*Bg4gEL9IduuV#KQ~dKV|szK(N$7a~|WtjMQ42 zF?y_3CI(v@9&pM|zE??(jyS%mDt>@aHlS4F=U#!f#=C`~gc+7?Enz(14s{WaUJk!B zi6|Vy{wY0@qvPYi&@<=V+F=d6=iuoCh=j`jjQ=%Z#s@C3v)G^=WB9wo{e1w9Y9@ON zD0Wz`4k1)qM+OKpHsk$sBUjP$v5h@4?oa^J#+2w19e)ZJD<3Pb5Ai@HP&uHoV*9(5 z(Ml_PPXDqD?YoRndS6UAV-YgF7OEtHl;O|=D#j+_l;qn$RhP}>>&K{eEz+2c(wk<& zydmQ}kZ38plCv!3_9*F?HIwEQQa%oHe4feXr$_naH20 z$4)LB(rYaqw$K`L!n&!bRrFhw(ixz^@a%5%SmPE_CZ1S@2kJlN&ETCy7mzK2?tNdQ zD;--ExH2NAlo}Jv8ZR9_R-;2Cw46Scy?Zw(=jQP`dI5qCsv&#{jF;Iy#lvAkb|3OR z>&9~kQD)!Ea%5VIX@T(l8RGFt?utrha>gX76zOIqTqOPIKqS6;?3KaT)w4;W2nOjZ z#oaJO`--x6RxRhAjYOKY@Waw>0DvZf>*?9KprgY!?t*33E(SDiB9nZ;(;w(XiavS1uxyK`XoR}P#@PZe|; zuea^SYwjC3`llB$^6ZnDO1l}fORv!~vhlUB5+;)J!vueXroo3sF>d>8%;gdPzHh7y zi0{l)F!Rj|_=@n<$wwq5l}7PRy8`!|dac|(L3BnNwfH{~5+bKw!&E<~cB9=r=ofn0 zQG#XI_SL30lS)j_;fKz@5$(n?P?6e}h12&c2D%s}Y6G1A`J1t6y#(+mL9L(_!m2u#g<{v5^RC#=l8KXK*CmUd$22;M_`>YX{3&?2UfVV6H?!oIjg`-P ztSMH5zge)}6Srsp=*z0xo4EyeST0&Bq)&ystx!z)UXq@?FVta4UAT-fjMqd+YHhH3 zUa}fEBxL|}0Oe`SM|3TunAAdjc_N7n;tAeWGX}Ij5)uiln*ndvjX>*3%FotzUIjzw zd{s9Wy|2n{-=<^9ymddy6^TRcMSEaI43BHdh;snE%jKaV5$r~Xqj=EzN=WrhsFWx~ z5JQ7S@~I;SSO#=>Jx|1&2#ch{H)B39PrAU4l)3Bm-(CApD;l$RTYav% z_{tX(Mu|8Gvn}}=vHC>eKFRyW*Wb#!2yd>(giCjIw{lud9#3Nn<58vD!p*Dmu43OH zcE3R@rzt3b2L2qK%BQeU^5RK|iy<&LdWZiy)tWfPff8i?baPoVC~)`8iqArbz>y7? zA3m|je>Ed6h7One-oeWeQ}}00ss%xETkUc~c^Ds!T%>fcNpfK6>H^wY;C_enIaCBC zRxEYRWlUS7*n0TW0O49h2`me3nC@%#`_lC4V4z^1n0I1AUX?{q@Xu!{vG?!c z`t)`h^sb&$9#!AbBXunKy_`iOxz!}|Una`UxIv2Ii=_@oCC{DAQb;dX6-<}Mp!#YS z8Tb5Z7M#$&!ccghVbeudrTq^n;5PIFJCN1XS4qEw^Xv2Zdk4RUW@B&LKiX@orFP#R zJ@rjG!4SS$Sp@4;ePEAxM1C57tmmz?4|UbvFTfEujS=yAMwTz}k>NbekX2rBk&zw0 z%nz3+&iy}wTCXDyH74G5R+ntYSKQW(OQP;dioB1EYGJiGsO28&-H@{{K^SI(&Ap+~ zE@VBsz&fzMxyBK2&lFFz#xgOGS))nkDGeKyQ4kuDlfulq$|lEp;gjZ~?{=QpPh zof~nOizymkN!Ol4=+|Hp?8m3MlrjHv@6IW&zN_n379HRrZtKnOsOsMoJ$BpRp4+iI zCWmFtMNGcDee537|LxjTK9QyRJ9coT``c`PS}xC%)X>EAcn|jQp?B zW2cYasrz=yN7K@=^5l^tKc-EcPJCTEU2^Kz=7oEE8VBUHwyxt%b5DN%GOGS>yECle z#_8N437#7a`hQ`>c0!Ce`u~Ozb6dvP2x(MD3%7M57xObe480=Ebc+ZGY3atk6Z5sk4ihgc$MuxuNsVFUPfG zvmlrIUi_aJ@xhuz0cwZaFn4)^BmEz)ZTRBr>HmchKSxy?9)2mrh!0I1-BYl`{eQ=Z z!wll^^=fMmx&H@7{NefVv3D=;jQ*H@eEokg;;X|6qY1_T3nSihaBsH9wJYD=J-zh* zD@JVeIO=~e;_lbQmfzQ3mr&rEZ%XZCqTiG`sr9@mcQX-U#8d~()ksVmpSeilNRjnPvYm*0P$tJ4L4xo!01>SV>dt`JUPJ@vw zcAAUy!!NGE;f0t*<^3xQ&m{9sHr!BHv_h%w-zyjha?(d+x!8vu)Gd7>(BNZMti*(C zz<29CRJdv-S6WoCG+Fw_x9RY}GqHJhQzy47(yzT|q>V_)|0NzMUBMHIQ*C=>vF4t8 z&p<~bnd%WEHa$P$l;#F^Ri?O|-I-+R_O;FL59PAF0P}Vsb+YuTSAfLdy>pps=AN%o zkKX@CwEFh(y+hu?e{QcbFO+US6Fm82yM8y*@AHQ%nGYv$&?3oezvhtt)&t5oR{!m4 zwne-L*Ghg`EFsVirf+oUL-Z%S+|v?f%QQ8%#6!i z*~%srE(hMN;$q_0;hD|_WRY=($E zS)IIJ$W`}jY{m9OO>VheB%?+{N4=FD#3K^2@Jo#zLunZG$cySG7m?N`KW=_=O4w)L zn0NnI4e4Ixf{|Hlje>}cqx^SCYO>DEri$Z)ncT&KKtlN1=N~0cX1tD-o^yiwrOxC} zCZv>f+)->mOfs*+PbwUj^^wsiMThhDS-%4SF;gyOWwpeA0C}Y&wBk(GebsW6aAiMP zN4@GZwZ%HEDt{mC&GBu4ST}e~8oiwSMu6ybrN2PqGSNSq)~ zAh8sW1};N1l>t%6;xR0}-_%uLBS)hcf?VxN?F#7-$PQs;=3JRni_}OSW{U*-TJ?Fi zRpa=M6(HAeSn{;*9}x(LC$g*bzgE-+Fma`mQJ(18;r6LhS5ZnY;lLX;6wyZEZpwUB zWU*)jc?@s;OY(SfOEb+$976hHTSF4Qc?0_lrfbWBOePzRqBK3NY<6e9nCzW@6Q-0K zh=G$p66AP|0Am+6ndHzLgD&yyix(+ENw;DNDmyeuqv?7o3D=T3DdB-mN8_?!7PM8Y zc?-i4oP^>u2}xUCwgz8Fctox5{~Lo%ys|9C;<1&McwksKP#-XCx69}`LJd3AEp4_* zV9n~~#4k$)m7-#_6XBKu+KQB0%WGftahRRcP*>P_lvo{*OD?p*h;4_6(5N|?FNIlQ zJR9|{7IIbpy8;2-8he)yS8MSF>F~g^!;~CtOe^$42Sua{fZx#L=OA6&+DIB}7l-ToK~s`A@fqn5%y)!MRb;kRynI3MvIe=Ruq(3aiG;mqZo^uJR01+_I8;$4ae{Wud7?B|g@Qexa3IiQG4;G7c z3R9+(3*cP4ep`bUhQY9Pf;I)Z|124kcpwy8gGg#wrUXQ#P%ww6fR`WIqdtiO1W>1N zGQ*Xk|M4Z$==lm_8J@z79qEituB4E3m2`ngG_30wF@JkM_7{S_3);U2j$3o$4Ue^J zK(t4VvmG0WQ~Tf04Vp%DZ=*qyjZt>H^q9H$`QwO8nSrQdu1rN53k~Mxfm7m<`$s<| zlQ|ROlR@K$ibX5V7v)0;#6$YWA@tfi%TvwMlG`O4}`PV-wFB?P_OcnT46xZxgK zISZtY$x@x~wT3f(V45&lN)90ivS0uloHn`ZJ0@yctL1VWCQIHaKig`O*ViVHaTfW` z6)C%Je9KveLvR>lttTITgnPOOl(o5>wGMv^zRlCvqTWx?zoKSh22wByLD!dbGJ8rDG*cokkuP^3?+p{}w1E5fT9`|Z_C7EddcwIE5I<&ap zG%gx4tYH3C?Z@i7dWF@G{~lXevbg>GT_Cqe`FLN`n=fvS&Ph9h=JtDbw{#D8n*7thqpILFV?LS)6&RH1mUhYQ7OB$N|Hh25c#)e@ToS2he4Q z<2n@Dna*f^6UU@#?~QK~>z!m__Au_XD+)j~61{a8X0ra#`rjkQWDj4cWT1{A<4V&C zBc*LbNajwA`8R=!*yH^Ub|39&dPF3L$o3(jvbqi&BJGzop%+m=Izqde0(kL}3d4|c zZe-mV645DJUo8_`3(_ymYIp#;dz0D75P_$8P1(`<>_8;JcP#gbu0HLQ!vH&t7?M$S zAYYnMsgW+@p zjI4J$+0h<*f^Foa^~eR{wcqq|fxgDR`Ys<$`xB^m#=Y)+ZjW5VFpypb?B+BWlxExC zbs*cXs0bTV|KzCY7_4JHTsAV)i|S1#B`rKclBvQT5JL7QJ+I=x{>a1J38?!6K%0md zrLk`qnduv&JRj%@Dd3`cq}?%OQ#>+jH3XXAtCoT+AtD0X4`3OPv@Jx_Jo1iAQPN{X zTs)+RkGw4ddF6j<={39&6D7B3=lWeDbFK8N;4(xgLgvOJRdylU>`_tiK|3^~=*dCZ z#xT|vf{_AWf25lrycTd#6**}$H<3*YIC=7*ImbM0DmLjc$b!C)xdw-B2WL+S*#Jw?zSsSPJxV)#2qC@FAk`SO$3sXScJ=m<}ko zVVM&TnIC~{ASGRs^(i7Ei|9xrMxihymBaJ2V>ih| z37!C|eE2uBBiXBg;#VL!)55O|pLB#;Z6MQl(8iX-xq&WEDnLaHxK4av!xkb=EjfW3 z+Ym33Hk@k85V1Or@QO!#A1>VqVG`o=IA;Jdx00EY+%Xw^&N=hp%|q(h-GmgR9Tl!9 zlEF<4#4IA7UB<=<#a#!aKDVg|b8zr@nVK`>OjV#F052z@KFBt~WR6#ziAa6uwmA!H z$N0%j9m#vFt2)1rb(5a^(Q~t9XD%pevsZjzGvj`!lOA}#EtjNSKB2B{6m_MZRGg&}|caUoQRP0M(rzg+k3FxhItMOP;twP#p zWO|F3F!A)8hj0&l0B~^P3=qdh3U$=Nj4%aS=*~-ETP~s~zB&#M3p5mF+_A5w1hYpW z=ggoJXZF)tu3WmEd&Vvfa+ak<;>PM?mkG^k%_zRxUXk3VMgR`6DI28{u1rY6|)_hx(4#6B+8q%d{ zPg&DVzZ`By0SX+CZ6YgFDbsyPh=|gN0KQARTzFkr==|Y{yr0Zlk2{{7MKSoyV+e#k z26E~+axsWk8*)O(%GTS%R|aP@v=L0zqq)3U0T+?Wg{X2Hnn_J4K3HWUgFr;)a$^LG z2$~Q`<{``HGxRZME;pv1NU{1l7i-4|j028Dlm=_JoVCS39yd(kVr#;-P9-V`m#0Hv zMC1_+M0?&OHGZV0K%{0hGy;<;79Y5$w_z(4%o|SMVTlqJ;*}bcO_xs!3L&;##RTH{ zrxw-oy=n7mw@U=|eP|)P8$EsC_e<~iRP4%R^!~v_@bQtb z#`9|fvsj_2Oa*tLkFe;2rrfqmXm8uD_9V5V6*sMf)^>O4yll1E>=c%YIC1vt*}C+R zBRu3DCxEtf6?Zj-MXFSJWI}HAt}-k8VF}wF!9q!}Pot?Q45T$^Ok)w=EgJGYYLCye zV%8+w=xv~W)yb;a82C?#rLMzOQ1|Gl>DJemKMc2j$`}F`jTB}ub2*$BG+K>cc=p*>04>?}7mn4N(KlTxshI>)+d@>UtsFo| zfh?zmpfwTill1g+y3K#4FbRitU_@{ygnul5$91fx zsp35oR5wol=%X1f z<g!1e(CJ744}2nj`9 zDv2sR+~@{vXTVw^8$R<{+T7cp-)4Wun(M@K&SFEJyd4t62NHnZP%6BD3t`2>6uV0{ z9v}Klc!{C_!pwp$NNL+3*x|q(9Q(;+0QlXNH^;Hda#P?2-Ziw>81DT^Mf$BBg^MeS zb+!yp&>qyCzo&m}FtbNe_2vk`b@9h1-vGWM>iI91RBhK&Ki+G4m^=s+LaDHl@ zPToVT;kSYb51V0rlY%`CBFJHs5sDo%PFw>;{$d>-BJy6*f^$NZ}P%!i~-}c?rkyO$M z_<b^Kpoj`Qxsu$KPY@Usv(JCt_pv8q!wtoyYn{G83faq=4rovbJ;G+nPt<&=s9LClg`#9~=h5%UANV&Zrg?blm#m)6z1bYOda1m~R8H?X6M9#j7))RGM6M?$?%b)XiR(n<95dF`G&}VSYnn_WDthXyu)$(&Gf7Ce-#(QnaC;z?0e|ENH zp7-vazdHV*S~30*-h0$8Sf*o1gJ*jSz26yA$0yicYP&rXmT>GGuS{3b^NGCOi0_(; zvqzGP21POA{D#B04+Hcq{#YMG*@5ZGd|o!=0L&MBhHO z;m_Srm#uXMZ|UKrSn=g0&NAH6H_xU|!>KNLu1ar=J!Sya49mW)e65G~BH#G&Q*_iN z62$ZtbwT-?fx-9Q{br#apj~=(p(;ylxi z(?N+f#o#j1Bf4E%BKhzd+R zq;OxqUgB%F2PAJE%o(^iyk`ok6cOs8yw)MvVHCrAU9PmrfZcN#AFsf-+&OaX#Tysd z_5$;DzNEEVzs3j-5zmws%v>o;7Hs@oMYMQ`bdPhk^=#rx=W2c)Vm2GRobL`n z57pJgPVI#bg9P?75pt%$5zlglsNHawG`!?t_F z;`Clwz5Zmf+mnpH>!}iwmD%BzlhMYOPl@L_ES)`Rr#(h^*lDu*6DBGZIJeiIUsn;^ zXPXes+qZK6`VZsj43m{*nn@}mPj>~A&nRlu)6as_`O3-5NJW*hj&Vtl^0J)lK=@Ok zm}*l7giUJHlH@nCZmZ~XvzE9+ zZO!E6P#SD2g=0fw0cg34Q^`nWlPF7n+tf>tJu2!+X{C1SwOoaq&<=<|W9fU)4C4;L zIq0RV7K}3mrS_ZktCHt}(l; z_eJvP24&TLa-R@-XeE@7Q20jpllfI}b(b$$Cuv3l+g=jU=+HZba!f>l#(7O!o(=W+ z6{#B9lyusR(tVF?jujOUDY3@WO1|Cd3M_z~+2;k<7&D121EU3mXU)_KWx<2t1!-$s z%)WMdEAC+@S)Xp>lZL3k$1~f0T)Mrc@%(X%LwSd22utm!p*oCc*QtJGEc4b#lhxQ8 zgT`UdxYkxXt~~4b`p?|0+B1?m@0F5r=501S==tl(lVKPpgdfj^D4YX=P#O?TEos?| z6r0Z4=XV?oi5y!lEEM^Q)@=2-_zHB#`|(8`cSIn)ENkhBQQ3k5|C}65 zCwLj9UbeS?>qgQ zSRQ%yMs!c-oFbvq>+^ zd{=ayTJP#(>^B{b5lLUsTrheD(hKj(#*EpjyGZGlTyjkFU{^cJF*sUPP|}>u?e9Tq zi0-7|AYr;5(Lt#e|JG#J>er~b2lehzR=FloVbqSSQ@1p;HO%Fw>NB@YjrQ(miK)uF zm>cJ9UZYLA>x620+)TYu;wnNSP-c>A*@nUWaXyk+8YKl~7W%Jm~9uT15TXTyUe)|Gzs z&h4^&gr7%>vUGySW%FX3K?Jr*dfBWOf8=8x3n3|UL8(Pk&y znIQ6;AmAyiQ`=uspnJ{qo^cxm63=*O;Y*Ze@i}Ld2M6-!c&BS=t5@xfpnUWL4x1+| z%gV{96pUV4)ZlURxpYi;K6bA~p43d&N@6}*xY?Bl^5fMG#g}VyIAwXDL?vW@ca35# zEBlk~!COT)_zE4z&w*V^`i@I4Ki%;?CjX~!pXP57Qtde_5Y$_^Fg`U}6CRy+fU9Vk z%YeRgzoj{&=(Vz_;X@@mjQw3F1%)phg8qP|9ZJ+jiqehH9XnrGKAa=hBEV!7Rd7GE zD#V|%y*M4MX1(9oqo2ni)*2x2e5rmZR4p3t7#^(Pfb8oF;1saPZTx;V`|`xziaf~7 zPl-z-qE(yq`U;|g4L*TP+g%iOqj-4IzMqo(p{2N5upF-@lLE0L?Ge1o$~ca^`>dqwzF6-J z!|EyI0npT0nw3eFd}r2f%U2iyc7SJ~=Ih*Q48w;tl(->f8=ZBJW>sEZwr#E2TiYZx zlHcWp-8%<_iS8tW*d73+AW9!(csRdmUqo^OjfIkF* z$7BSEv-t$;d@f^8BD0LcFV3h$X83C6Y}K}RyNOOM>wFIUeU+q1cqW%drA)X z7UK|r`O_-#QfN@@&M(i{TAxg-@Oe^cSAgfwjbuk<*F~C#d32YDI-V0gzMe$07n5GW?J;_ja0S~a>AiIVT zilJck&e!?WCESPxsEb)GPRR;m7w>REH7h~>#afTP;gB&G95nSjNY~->%n}8h)N$nP zr%cORz_x~DIX26li@A$$ke_#KSuCvOvD#-833) zUa1ZnpBha(OOrOAFo_VFiazmdaT*hDS6=9)!qzZM9%=r#dEV-{q#lv+mgrkC27+yv zL|aLpSe9sy(y3Z`^2D{ajSP{YGez>)*+f8w+F?Hd5>SSraoL=u4z@fVOjW5nP`efuj>NJ}%hI4?Z41~Oyd4-*Y{g?r;lb;{1t0M&;rmC% zKvjXIwz39DriJB5fk-Q|Ja{8gbnxkpLK+akY8qM+tF@>Xka-vz*wFnP*a8Nb z*wuoSDeNozt#w@8`F7Kae6V0dzm@R5dB-M$md6t3u)<%XI6P2mIm?!hQYNzW^Vx3< z7c54ey;gnkI3`Pz&e9w%ma-~*Mpd9suuI*C9EewA9!+({GnGhSh3<;ZixQqjg$kjJ zBrn~5AYXHxAzymIF&}A3$^P#Ji0y1_muRd_gsgh!^>#OSU$gVX^w(m9n^p3jP|ACs z=CANTCQ(p%h3jMzOos>30w5cF&5jMzJ0n^4PluJZWozC^W8hiYa?{`GOh>XtYY|ut z;c|VeR&$ssvV3Nks7&<*_O&zI+Ca#|C1?&ra~4qMv2#IVa${%4A6CJ}A&6svKV5cz z`_NOg5)krBNr0jh8E`7K(+Ymu6zS8L_!hQ(JY7_!RgnWS;d=f}H!z`r>eA8D<$Lzb z=z1sNyITyUqt1_`k2KNiamKaNRNXPirtwn#q`p$l3V5UA0fz8#N5NW#b;0@T1cmt zTvl3^7o#^!RaTkn89ffMTI|D>R$MfLmhmM-k`%8kw&J4pxUOgE2F-jj<7YPB8_~$W zFajN1hzVR3WsDeA(@l!*x`v;pj0B--Sq|$^UUR<9;qm0+)xs|+4H_M&9FOS*&elO&<-5zq*QCl&6+$z$w<<=rrrOz#q6KDsu-2aqeh zkh}zv1_4Im8>9LjGEvp$GM~S!L#Xm5=P?suySm0Zr;HHGk;>7Aub!E0z zNYkuAA4$Qxn(3Aq2`#2Yys@mf3$V#h3xB^uX;4Gu!ODC!9ABnL1binH4=lu7*#p@i z56;@gRGBv%Z^r(9D4%p&tl1g+u(0~$F!;pBo)`3lLCGv71az2FfCCDNCG5YP0yP?R zG%!n^#&oFF2{`oDHXfH3J#h&X!ZdvG?WCl^Hhs|g$&zYK^2SjzbFCRYa>g7wnUYs3 zpY6ea83uziUa~s)sH$BEH<4fht;vW-Z|gmHH+x+3eB_>8sxAu!St)IN`@lhyIL*d^ zAFdcQX;z&6(E3fTr@!TNeeuVJ`x0BzI?$A(+=f|_oQc_bk4+vGO z8ji|9NJv4^G-6k@`J?{Z$G`vJs-GOWDn7E!5QOdDd8*AnQV|rkTu@etYEB^CZrTbu*zhO z+url^C-SABMN+W_ z_%`u`cB0av1*7hbhkGT`-P;I68_su$|E{U>Pgl{-Cx58%Rj{PwglW$J? z=Rvw!_?EB;V5R(^uYSkD159VN;1D<58To`$W5$Zva#P=BPDowx4+$JodRPlS`6&6c zQ?YJ}EbEvbP%Y2mVrbC_z3PMc)nb!i1=_h}hh0Y7jumdokG7wo(k7Ri=n7gRDiyAM zL(6nqgAJLO9$lxvilmDSm@;cpF#p89El~Ew$-qsRLYz(Y?DOp2Z^nz^BH~l>RoivP z&op>`jNsb_C=CT1QbpJx32ws)vskh2Y4v6S=XjGf;Yh%f0Mz7iUSd+Ilw+#3fm|iA z=Fec1yVwz|>^>a08YE5w?$0EO_86V1OuYELHmNA4`)qIiFIanzTY9Y|1PZM=oNZLe zblPPX>pR~#7UXy6JX>*^MJ%H0!*;Y+h7kL&CNm(iMTq|KOuqk|_UMNzzxA%Vr%)Vr z#c9C0>_Z?-0jr~GEl zFRDWnjiek;hu^fwzG?bZAkxR%+MJ_jFAO!7g{An{R>^q|u7A2xef6&-mjSmVf~C88uwKNe z!Z29?(%lC#;Id?l+<-%CSL7nLSUdg@KMcLqIMq3$TRGQYi>^5E0I+ZrX2J+ugRwGiXP>{)?>+!JKObd6O|Mv(Lr6e6@A|iR})M z5|jS|tA{bxiqw89DNByDjVek8_M2JHboCEwi071$Ra`{Sd8(*S3#pOiwQ9eIFT0&u zwb5yOhC6Os?bCVdaDk?Cu)w27G5@cviH{Iqh-KScRE;DWAFhpq%l5MdoGUNnxx?^O zrX-`QK#<1VS6OExOe%sHjPamqT*;>O4&f{b0PkoktQ}=(gD!JaqwH^h%ZiOSAFvWHs+Xn{?PB9>bPP3xG|m_0 zGP1RUmoJ$aKhTh|f4z$3=bZjmX;3ik8cFM^%$af4i&Gj`6&4VT+@{*pbkQ#RQ!Y!Q zcb^l)XKQMx32uZLJV5l3Z4E0%061@1(xy}_%7BaTx=jQZJK1BNj+9;yK8DxTa=@q! zd!+b~1sZvWQqRvba-(zi4?eRopbMW3+K}+tsx~RG=ftX^6+X&Ft3F;qYO_;cu_ppf zqH^3$4DEKroj+*6gL`p8amDKe1_5?pPn}S(>CB`udJ9wwJt7e_{Qk)9x`hoJ&l0KE z(fiiM0hw@HoGn_~nG%BvjGDk6yw)vWw_UA?2aC%ZI28u<0?xzMfUkMXq}n8)W2CH| z>%&m$XNrJ$vP!9LAqr&(f6>5ANh~K^WQCV2I}>@{GnOMKc1)HuY=gTq?~v5RKv)X4 zm{BYe^o#-F)h5lp*<^WKRc}m0nP2XyX>is7%M0WD#?b%fD3k{miHkV7@f zu1!=|y%kO<^ePIT8ONV+I%k7!lkQjxxMtHfFRz+1KBPfMHSr9J+@3Xv+u7oEhWp&} zjB86Z`RLbA*#-%}Ac2;l(6sXuc)UJ(CxwoiOB+Ar*XXtwyG%GjVnoSOX6A|OIfF?~ zVcdB*7i||R7BykG1~5|8>;}|=ks{tqQQV0k!+lW`=iIJb>yV^FF4B_e{EhG&Mjq|2 zr6=6xF%Y~J3VEO8+w)XY0NM!;mCFpT_DlRuxU=d6R}zT zJ1y;O`@m=WQVW9t$TO;a%67+1H`KCRdz8wi8hlam=FdR^HQS^U_AImoRM>7s4y^=+Tm%u@M)Z&2KE@_!sW%&i-&AYHxY24dUs9|VGs$M(bXZL#g z-x%A(JpZh?%4@lM%TBxcKZ&v@bNtcPKli#V6dGPlu-bRi?|?y}J#_l7r+t-@!57O` z0~k?!f%lI3yd1Cj!rg@Y6qNT^rJTAZd%1tM11wZu$~#NzY6Yd)7%0hx@*0fSoTq44 zF9hV<-QTOtv)3tf2Sqlgs&<)FovF8BxLIa^n>Ox3?{m>!$0%G@Y!_&aRA>Yc8>Kl6 zHSaRUt+v%TdQ)LE8k6%v-olCDJbR+gfUuotXDP<#omZj*-<;sJNHTa+33CLzO~nY%Jo-k|}A`9gGpAv=;L*-boTcWny!;0%{%j34(PEm~JX z1B~pJu!_=*x#SRqRMI3R76zo)pYa1ksRG|nMY|N0$5ToTyTc)iQ`RQ6;9JQH29jgW zRM;&m+KgqZ#cDam&c)j4Hbr3v!8==vqcCV>W6HAzi2JUm!Nh!t`JB3TE(J}rgCyDD z46;Op5;2{LI0y5b1Vo9Br%2gOwUdjQq0VGQ+whwdVw;k9HdshW7S7lVTR&cQfzQwl z@e@TVwg!bS*s8PpW1sBCLsa~r8h(Xf4o}P%nu48lj?V%t={8h;K#R2Xzz%*A4}lq@ zIQfVgP(=ShV0{f@MWSL4@6(y~3w5mu&2~OfQ__2?Q%E4ufmuc82QLtI&F(NWe}1mh zE*O<)ze=O>dCrYYllKB^f{UEOJXmf!FkYVmnFN!fc#y1dgANLmK4su;k7NYNsW2{_ zkfz*WAm0t_U)Q2-_sq8RXQp;=hr!f^-z+3KbN z&~P8&Yz=8Z2+&cW+x8qDlrqRe&%ZiUm4j?KfcmRL3=`X>OS+Y2Utx1#+#2nkHW+%d9>e~tRXj}CG z0WpXqJk2K5s0;PMPsD>k(!SQX;4y6Q81BSt=a6T%@0li1AcH@wsjs2NG$m^;Pj`FV zREwb#&b|YSG&W*QmlAc1Z0RVrJPj-HD|jD?M=-KAA-z>Rkz+FnI`&ab)(E29jfEMp zk`JiwsPqvbSsT6wiP?pi47XWtvrN{Z3bp)v4n-{w(D7x5{sYTiB|^w^WwWeM6Nvca zP1QdTNuG^Ku&n_Pl&B&tp7u2Imkr@+h5b1tKiw?(apoG~Y4SB7TL`xD{FbP|P9unP z{Mhbb_S5qe$T^1oa9w5+52FIvan6x2(7mAF<)r}Cd=J&(ui0>b9T>qxvf6|Oxy8PN zc-MYH)u_=vJk%s!8O9q^zwhLn6krnEEpG=ACZ;bI`HX1-aw@h){9r>n$gu+i$UH;8 z)<#6QYt){7|L3$gRq<|MRCBV8p3!)ul=Xt~1lI1!D_=k%c!EIPZ~C<7kO8p6X$gy9 zn4rBjau?`~8Ry0mC+IfhPP1{kz4Sz#HDIzjinOF{%AbTT>CnPQw?5gZ>A*$F z7?x}8M}|Pw?Q|0iJ(JcEdtlH_D00^2RLHh%`GVl#$cjIKq+1_!e zxFZ_<&{3nqBv}&vkOAUITEhVKv{A#LI^ezW9G zTjcEWQATDmKt1Wf?cChE`|IybEEH6$bj;9BVEJxjBa%;N(jB`-pFY0nyke;-1vc7^ zZA#Nt%8XbGktl5CsGkDe1c3z;q$;7?NE(ia>2}}1n#>VHmb)qSVU=dnLX146X265I zQ$R2hnEf2vB;jY9QGAwe(lGRr{%)bF(ov1^O;~}!PVhz+1INu*v_9FAy>JRF`kfU_ zyvdSFwU5#kJ$n!vK6TgoK2wC(Tlm#MVP2xOXGHc`veJ zUu~7p^~QO+rEPDLnygnxH%dMV%(#RE#cjC3^5>_T?SNL0jWvGEy;OZeserg2q+G{v zEdP*|HMXf#W*8>0NKk{UajZBaxULw7A$r=hU9I~{5{t~|@LdBE419nrI+ zhZKPPFOI6`(lFBvT~KcvF~g|qgty9 zaa2HN0-!112c0kd`9d-HFz;pXP@6s^pGLhl#Vc<@*NkP;rkHk#PmVwA%-}zSxHXNH8%z3NmpNgl8 z;z5951a;rs1g%NcS3Ap9JI=5jr`btBW!L`26>9k-nHC}23(FV|*fFe8{~-!ZFB3vy zD>PAorR8|!n0$e=F?r`D>ZdRuF{>}DpX;y+o++XwdybLGKKSaYs-*o}KA_)k zM|%R;D?$`l|3K3u0j*s4oB_>)hL3dj=dY?-3JeE_TkW`@p!eK*tjk#mxa!?Z>rhjzC?!R^)PH+a}_9l@_vwul$W&?szga2p(_Pm zD4?-a6J!VMOJfm(lzkKla*C7YpTC(YkOqK=J=%<^QF{Uvwwi8t?!&3|`$0y~8lAK5 zWT5#9i|n`=CLGYcs{bV28a8FBsaI|D5?u63Pp5ZBJf#a_F0uMPiao&1MxC0fA4%gSA{2IoBo6|B^H85^is8hWf($T24B{Z4psdw#2kc61Y!Su${%V(k!B!A^i1m;kW=Cq0s@pQ`G z>L7-kuvHF{+GYq}7gshcJPdaL2eswUD#>>ll6u+X$O`XXreZ4X`OOaGXj# z_n@r*&B+3#=7JP&ERkOjJrmTvcHYvTr=)8KKnmg*>g}+!MHvZjp`!^~ECLx89_ z{mMsYYB2F5S=8P8VQ}miF+QE>%aePxf(+Y~2Q%`mgt;l6&gjlLxaxav->KSE(P^mx zjD4>F2FK$)8qAPO$F68B&%ILqhzE*{V84rA)Kkepw953>ZqG4H+cKDl78bn{3b8s} zkf?!_&WcoF(Pw{n`n^tVhc=&r03EeAQZG$hv?4H4VJrt*4XO&Qb`7d}j-_Ic7MQ%p zaf};OFYHNZOzkW3bpOK=#l5#DS*M)8qc7v@c>2DgWEU9pHCzP-+&;$FAah>Sq_tl5 zjIwfOvth_f4WiB?%*DOiLL}5+UMY0>YkksW%B#ma1Ex(p-Y*S(y`pzsb@gO|*K3K! zKKF@z6ZwHZz3}mM%(av9-4+C6U8~;}Aq1ngmI~ex+&GUPIN-$FX{e+y4uf5_S5r)&x2d**~3n7#D|&AHC+O8|6^b zFBWKi4r);F`o<@0PjwY+ud|+gRUuL~3TmegFT2lFM;}6&oN%U;oULCU?VdI%xChOL zuA8SSw=+w)kW(A+dl0mOkpdldRriEIss05(8L>gy^V=$Y2_U5!RAP4@X=?SPsal;y z>-FE!9{o+66~J0$_4}!FM=+=MfW3Y@#X*`WJ0{NuIe#IT+zT-~!$ncOD>b2gRU zHeF=n*>)nDqr9JUaAR%kS8u5a29vWgG*`PLo!sZ$*f;#PX43Jm@3UJCetliob@JEx zv%R}kcE>~wJxuT%_LO+IB*W=w+;}yZQLYIoEmw+q$5Y9XxEo@ZEmh2DNTRvHb^lDb zCpJ!{d~pn}Jp$ziaXipnT;27A3(4=uhSa$NGkj%t(zuWJljQM|f#XId6XQNXm1<_E znb^3Dw3wLDJGCNJ6L{ltOkCwS)fpsmmo69O5Exy;3QYY{3E}ckLA8~&a#XTc(BovL z-ZwVV6$w_3Xy7@L$*WI)^3h+8W8CeLl%yYxSsrjn>Mpv%yDj zQ!x3kh-AFQ$(;R|0vo+3C`@C4!vl1DJjsA4V`)?=cxZ$?5DU%zByzbSs%LmIgq}+z zoyX0)+92dXy%eJ-Mh%>au;0k8`J{on00^b?WqT? zkZi$|ldGhmD=mXQF2?hmta zAR}GzD+VlTs6$>0ph_VCJD&=`VFLhjC;_xV6e<|BC4IEx0?m;HDuHsZT40u^<3kU4 z-W&oL5IV8j`bSg;RzUat0q{&R3{4HvgxQHRZq-4c7&ceQ{{u)5KckrwtWZkDAd6Ta z(x6-e*>zzDdsEX=>OgU9m@*_5?`aSuuC1T#%en+P3b&I!8p_~8bdk+*A}wBbn#7^g z@0R(4a`@fO*bQz$Z^P8MTUcp8-_sp;vUh&%CE?w3y)5qiNZzsI>2u9Xmlyu{)7z>1 zzfI#?Ly8MFZ=~`SQng1PH)KPRSZTX=H@9<21orF?6p_W3+v*6{Lng8A-{LfvHO^Xn zE?0fb|J2ZzI&Qe}UeT^F2m&LaJFqg&8mN*@+|4U9M-wdMnweo0kJ**r5AntYZeI*` zys4~ePrw|h-k|!JI?lZc7rnMB9+6YLnIsR7sO@a%AvF_5)sMe4ANl}4zTZ1B@U@o; zF9jzI6%fZ0ic^vFVAKBOvUc25a)?y*30KjPdw>N0xi>{)XyP_0j+;VPRC5|;_sZG!{OL$3wLHE}Y{k4D!AlG^e3Y|_3}7`+(u(b#AhPuZG3#mC3CCDnYs(BXw7uq#Yz zh35UErAARk)8AklJZZ<(amfT6VXaLx8~cYuzT&Ui60F^K4;A#bv^A0>qJQ$B>bfx?a-IX#z0Pb@`8t*vs&J z{UBty1bW@#{g$s%`C)q``FH@n36JnVS$5Gf3%C6?{_4(b(R5oqa4FZ9mdf#2GcXh z<;u_V0k^*0!u-3`*LUTOuTOLdt__z=k);gFv-gr$)J*bS6hqu|+puUQL_J;8J5D_z zALQSnt?a3pM{NJ9oNz~~!vYoh!XK4;R<8MknGkj6xW^+U*kq|MPLdQ!nCN^_j7z;K zA^}4L9jjOxSgJZh*Bmt=LM7mxLH?&VAcjGY@vZCLtq>6%PxkQdBp^JJ^F%ZNxuQ87 ztFjXyX|h$4K}zUPuZP`zxy6|rIt)`noI0b@1=@_az;IjC&;X!!(>92pTC;3`E`%II z;M3^V{t9}q^{;Y%Z|>iJ5ZL6OXRv?-@MeIG2qX${Fm)N^AjvV20I%41G*Rk%tFQc= zk{rGPs5bNNdNGUtjk+l_%9&0X~xMz0s^B>dq4l)ZCziJoEQDnk$+@ zy6=#5NCF_{BqOa%e;ODQ$yO+)&CC_hejf5SPi1FkptD3tA;V%?bP|>^i(Dl&RQ=S8 z0?F8vHUvXZVakDQbnbEhSPaHV)r^W% zba*^MF*s6+Spk%jIO=x*^b}r)junAwgheJ)qOSJ{QeGfcz{xmL;!$BsG}{~W z^R0I&?MWmJYR>&f#72jW9GmrnVi=5mCdk_dDNFE{ajgPr2>=N3dqkskFD(;LSrTJ3 zr8WbiscXf9&2aJF zFxAqI@B!iniIjou-{DQv{stfvvS?|`5wgVSg%|=I(W*2?+l(T=5tLUXTHj`s6d9Cv zRw1!;tj;o4IhOjGr2C-kxE&U@frH*Rl3XXGZCf=|XM|l1(aoZ%p_m|!czp&y41CrZ zVjF#M&%09u`WgQ;&s*&oH((_?NtP3d7#;TS@DMxjKrbQb5G3 zEhCRfvR;XS#)^$)VzoxF)}cNp5^duKBIY7jDG7j#rYc+B0@MI2P6Z?cq?M#_*~Q?z zPg)jYD21R|V~v@osiXti0R&AhT_snH3b)6s@*y_FWej`sY%5fard~sbpb6@t1;v{h zb~IQi1TUYb!A9BPV+61b8{x{+dc}s1(r^knr}zT4YuVsB0WnI@a2-K(AyA`I)GLX@ zE4K0zE)*&4xp?#NfPe@UEBI`!F5oFnjRXm2BWl=fR_)Fs${xqJhPNLz8doZ9QRuMA z;RC>!oo?hz2#Xkq1F)rZ-2nxB%0nW-2w_3MATP=)D;UVy!sG|nFGHa~#A#z~LIEY)7M4r*`mz|HJTA? zW0rS)Hj29$o9?9Yt2H*tO*6rfPTL}mvu%vQu=$e`Nm$q=^6}hDR>YO&-dvQz29l$l02$*I@UPGDA2nq<-!Oa z^UD8DheZrA)m)6cw(X*3tIre@>ut<(29;)swJD=whXT+LK?|E5ns9b&KCNVN9G_5@ z@SFJl(9Dmj?9}}{qbo$+mC(zii#o$C2YZ(5nA}4jh1T)~Gs8}mB>qE<-xh$&vVp^* z#ziPmD+B7btZoTBz^R!cY2op7m_M7W)(HXARBBq3;Uf+;)-ZHDxI9(6fT$59PR}7? zn%n+Th^{$2t&kSfn@vi;TU68tP#-p)IdbR;@T$ucGTgK6K6`ruRkI!dz+$Qb<1Sd% zzfZHmh?RcFDyxa1njK&p36#mxs^XV#2f$4F0dFxi>r{~dViQP!)DScjZf{5nHXNiC zBEfH`sn8(!9HM%z7(f${c`F7S5WWV4e>bFrLB;Q6GL2m>T%<@648ZB~ZBoB()Fc#4taADd_5>x1!GSc{;6c;Z0uZ?~7SANB z4niUUL>9N9$0G1*5sc&EFIr)mWEO)9=pmgC*ZVB~UTZqv{gZ({F_XT_?M@>xX`Rv4 zc;lc8xZhO1Y%!#|d zF&~`FMkh;)yNWDO!pb3c==eCSl?xNa!G4cKIi zicG29kaV_7&qRj@X_xP6F}Tz6PWz7U=^43`ZEx?Uc>)iH1v_k>>{))?9?$5o2tAC@ znH;Pgo5zH!jb4>!p2__UQ|=p43QRy>v~AfYjO$zX2p1~6u7>3km3oN?bbGAk%LRYq z*xx)D$FdzFV6cxiyqyaHD?`+zAVsaj z1`P0NXV|LnPOxeCVFfX+psi71G+6x@#X*6_5}wE1yec0-^V#X>O^{=!P-yP(rnd$J zf;5_WpzbiF%f$I}r!}p0%bqxl!BoKjc*?@2*!rCTmgRD6pB*&eg-3n+Ro1Oh7(rMla%(T%FA)V{?77-WD91Z&9_BMfy$Z>$t-Cc~Ng~YQHbw zNN`D~cR2I#_3a-Qi>4ArQ?hbhWrwybeVoi#5?_V53_7|#pJK|gKIZOgIk@6-(eqRyz-x2VgcKHfa=O4>!~30j%S)N;al=1<@wDg&wR$U`kyD> zGybvs@{xz$pDSCQ&+fSX_k!WBtKF`-EVW(Dw|w}ZIfB*-4VdYDDv7@^)Myo^uLqKF z=Kq`nN15}tEIM6tNsIfVCjMr{HIJPMCNrlFl^-*K=4vP@FBHb8l#N@;8)NPaGoBT- z9@_b5N;~-VzR`zcZA(A;T&^}-(vYslkecf>W1VXaAN4;Q?)q$c^7BF=OedCQ`}|KX5juhs5$-Y#wuJrq>v?K=$Yjs8 z`os%AS9PyXUx~UC@_gdLSkLqo+LP=5D#B!)YX6SEn%my`-)+HD)-IFiN4Sj&`+VhU zzrokBT|aN0{2BW5vBdq(p_6w|*MC0UIuSMJfsfub?DLQMz9zftpLc^3#*Me~w#)iQSJuM>%^Q(8-!+N5x72EebJj1$j z=oi|dckx`pt@nsMk`x}5p;$(bh=U3ahK-c|fGRsS7PAp9%U^|W2W zgRz5%8CPFktWWphEpB<F+X*bJ9mf<;#x7E7F8u=EuR^@IQo`%n1U$)t{+wro?)o-6x zZ#8T!bG!EapLb8QCqCGIkGuW3!|>GbRbo!h+zkUc>)HN)eH@n}{)t&Qnfc`Mp_^=h zvKum1s21oQo3HHV8I%O6w_^t{{4@Sn;_r;iUEJnB>Y@uCCLUmm@aD;}r8alGlml{oLCINE5O zAFhu*R;=%Lv3F88>3=xdthKurIWK0KTxvykPW`d{@>S-gvw!8Ie`}th4UPKi@YUlx zIJW(91$(t^$oR*^7dPu{<2z+v_C?*bSJHCmjoR~azAK*}emPqDy75ftt>!b7EcdyK zXJ6g%(TV-gfr$86LuH*?cvM=lznvC4aBtxn%KTK5_4fy_AN(IjoBjN+F)jI^)%C0Y zeObHTv+M5lYd`*#y|{I9+l}kLeti4#_OH7)Zph^ToGj%+HIt>h!K0mQ*sT2`fsgeh zFAH(4m!!$c&XLOndRc4Bg{d+4rUF7u@=8&Vs%LpI8BUJdt9_CDqSS8QsFkkmmTZs( zyFSG}W^~cJnKql@4N7spKK_E~^DsG{ai!v7!U6f!NNjni&caBILe8}`NGQ(iVgO3 zw~;m0qT;mGcw+A^=OXhij*i1xv0%OE@BSA4#kUWzW$Sx;c{N0_^ycA3-ob)VaRa$C}fBgrft` zdFgS)yqEoA8^ZWTf0p(;y5E0a!`WzP4IPn`B$G~Df9QEnjP3_+;__SfpSPy}A{hJQt03W*8Tp|W@EX9Dv5y%!Jx4m}+wbsBrHxrB!wM_`K1 zBE6rZ)P0R|76{Q7Zjm@w#`OKeOoFRJ?MJ{N&MZU(#!mRq&{pm|fs0^)s9G8ZQ6Pvv zC3f>P?e&{Uu&e`jrQ~hokj_J$$g7}yNY(!!gLCMtvWWrfS&&CEJ zI%$v{s(XkG+Bi*TgpcnCh^WO+7+akyvL0=53c)e8--NqIp}6SNl2IULl8dohQQf~M zzCzEVTp71lo#JwGFzY2_3+%H&&LVV!Ui>9>bD;%vG}~jI z;GWF#~_Y2uIKiA-jaV__APCTMzu=C`Z;vD@&f!Ri$ zC^(a!n5j$GbG}!UwUZyiqeFF|L^<+KCKsK{=AF@rnkr79VUOl4Y*rcd5(1|%ZPZQ` zdbXMF#*P;M*W>!pYu^7J;9}y^#>tpvTE#g)r88ORZD1{o8g5Zd%o*3NI}XbAI9D-; zPGBoQ+ZF4U?8tp4piCOtaq4y=l`?^?-|wLX{up2&ShRCG&sVjXZ#dY&b<3s_KwqM$~tGs_glOigL8}1d*`2V$ml+v!D7(M>x+;Uklq zM#(xO+LoD&4DBv1xSvu?e(*jnp&uG);LRh|h%XY_h%jeQtEC6GkH3v6-u^1=!jZw6 zWUzIpYDCGc)!9dV_Fw(epe-Jxj5K7s1bpa|kq6ptJwEz&c-ZTQk*9uXQc3at+hZ1Z z8W^K7G+AKIk=kiK@lCk@b9;;Hg0;ZslUnzW1OxiTqAkdFsK>Z^;lW9Kk2p9i4YFwY zAE!vaKYJMG<{tIv7T+)T0IZe(2E|P^&OQ_0IB9`PF=h2(-bE0syC@Z_dv*m%WAjV~ zBc}X|Pho6#%WHD~+!KGP9dLDJTtA=tC9$)}=jKdJWRlw3u!VPvM}*qj>_mGjxY_*F zl+^3rpW$}vQl6Ck8sAR3Uw1!TQg_mh=3e(IgL@ly?XuHQQY~usrQTJk*XEvkC+6o4 zqR4hIMK%#@jfNV0a~f`)aV_cly`yWr36@A9vYjfpkK>2}rQ-O0*YgKbg22?Tw&ScY z;OU#fM`}~s2s!47Ox`YuiOrf7Y1eUj|8@EgJC%e#vUq0Nn|8pDrI#f##96D6~U-U9|Y z)_)Iux?kq@!C*Eh4;8w&IQsA%R5Q-m(~&bdQrG7EaSj|okDQpUl`_tnN#VY-4psM@ z4IR6jCq3E67cg@pL{=Y~KA7p3v470)`6gr*Y|&O9GG^E>Q*fISC4>RrAftT!&!@pP zUf=)Fwo(-^np)l<3LF1kxm8$%z`AQSyd)Z_Lcl>>z!EFe(~u2?{tm>pNijN}*3bZZ zuj!(`=w>`8Kf5HjG$*+nyvQlSVCibHbs7 zy&1k{3bo_z`{)p$&4#{l^vo&Wl4qxYHFME42&KcCibd8+P<$*Tw>)?Do{J-QUlRc} zcVTB_P{Jou&qyGv>Qt7;Jn}FnHCKlGx|+~PNf&zx` z!r{T4R2lNZ3+s40S`+u+TNvW=B(%04Of*4hE9600w8;Z|^ERia+1clXLDYVPAsOi? zB`_t(S}AII617_lKedI8lrhe{Cd2<{I#EIzVRYgdi}cLSVUXnR{*fiMa>RGNQ(A++HelALT+D?RXN zfbeJ(+(|if-sqryJhYvY9S(}jm~#`?waA-DnAD@dTLxz1_(@CeHttDm@ zTj*RN?1($m-8=Y;A++2UMwR&cMzikF*g+kE;FpDdO5C7?Ib=3KEm>vH!y&I1bKNRHIRsQ> z`SD5#$}VwFB^Tk?4`$9G3(He1X}*!t25$9kTAXFpE4Z~C$DJe08;{AjsckGlJ-@kI z4j}!t%mDYeb4+NktoFLx0U9i3?Upnxbw*5u!*72(;MM{_`wzhTh2vzE}l|0%LI@Hw>NR67cLQiupI2NrPf1V2z`C^ZCd`aTQ4; z`{bnSF5=#!|3n-m95&{_+WU`JO`;0RtKPH~zSA<|SwqWSz};!kNJ0=SHHihZAiOfs z6xeC*)*K0JhnFi3fLCG>3Tl(|3ckJoY`y?Kvj99AKmPp^a-!RneK2U7>ZcGF__?o1?%czu)0G0x;PBuIkoNydn;eL1H|xN+=10ga8$QSE zpNzJY2ZgzTf>&2c6iBZz7nV>5Dhz^aOFBrlZS;8%`KDqgdAz~KZMmu7=xSLb$wlqBgn1tHkAh16LUC7=A;y@Y+sCpat!6{bU6Xei?k_<80 zGKl5K`7@Go`lgPJH8<#jwy!-ZNBqKzO+n)X`>vXy3dKN`35s(GEXhG#OZKV_f-mL& z8OcQlOV#6LDhg8h!QZHCv9%Hq98N)JaA7;|S^q&phktJ_HG`TGj6J^|QUHx7;-H;D zC1EqD)?qMQ2lgg9v@r<&q4@COqKD89?(I)TmrjzBFOb} z_6ZXkvSWavzppk{W0Tfd%GmLa|4K9O1Ka~8Ir~tNK!LT)y6i+VOjVp{4?XY({{2-_ zxJ5%-zZnG*-VtU@>oM$Uu{caXR-_)SEI*5rZMrfucr*wMm6hz9j=tDgcmg5%nOe~o z7tl!Xn&owLiOou?iAVd9doG&cfMCqb@!6`gP^_T79t7<_voy4mEg6WNvDh(jjM!r~ zpAS2!n0}!QxbYMsV;=Mn@|R+SV*)nhv2%XHs?@siePmDlin$91l^|0u_%w9RD8;(& zY}G4Ni0_dMEV5j~v0aErw_&8+JQ(rjvzB^*K&}j9Q+R_u|_N$m2J9NVaYjako0~?w%_5Z zH%VQ>vY~O)WXb?rLy+-k7q}%v^3YnJEh_n{zUJIft_8tM)k}a2(lA>zr7z;)mH#fB z2${rx{1%NSz(|F#iRx5GtdsX1uS!AzjvKUYZ1dlhohy4fG9X={bJ6-VbbOkLg`}^t zU>`r4Wr&K=Etz=RuqWd<=rH!;@_Aie6x@vttjTu|D{-OU>S8T`fQyw~QLl9B&}Z|6 zK{#6TCJjV$xdPnwN$f|uNID{(=_Fr77nvQ7Zh=FI2yA zLC3S|l*@XE7Y?cG+KtC}bv@JRB8-H|&f8crrLXS({GVG~4M(q(if^|LeJsj*|0Cx5 zEAv9`***r4NI*q+9&VPT3_<-a2u@<;lLYhmgbcA;Bdyep2u^wk_|ur6M$j`B{LDIGmp4M9!BojGApqj{5*9-5}(tUJ1b39dM59aIXJ`|yER_)B0L*3h7+B`SUmGo!(i9=7D zmESK>iQ5B0MJdsMZaX+w*HMWE{&ecb8I2>_w~yzH-k2##>1xPoenS{iWO1nb$qD!0 zUO3eIPgmpI!7b}s_4ZIX=V>zeY5zAf+T}A%$qw5MJTH}B0LOFt6A4{Z8oF~3Y%M$J z?{&Aa|4K&B)i{#_+dq#tmR~EI)OG`PWmUNLF2qyUkJYzE5+_k{i_xkq?(&9nmw*9J3Uc-%VgxmBX8)*_4C@3b*6K> zjAcj{(Z#hdE1%b!yW6|h-@~Ebk9lMg4Q=lfTtpltbeXC99dJO`8og*xgK-t{GBkwy zzYFg_$CkZH{Pd{5Z-(_)9-ma64{I+!8~VLEhPCz5!1d&ryr(ZwojqpNewEXH2)T4I zlYOmDfWGE+q~mK!=|)(##lZFTGL z!}i+T6SEf0Pa>l0n`KFInP|#y=^W}=1Ig)Hb)wz&ouL1AXmgRDUllq^TtaUhZ%0Y~ zv+r*!zox{2o$fzNJdZr!mtyd{`RCz zA=Y#7E-vEIpXr@fPPkKUV7B&igBmL((4Ffk2M@WNhgKiFb2{o-1SHr;{bR(n?;i&> zb+x7WcVFH5s>l_e_9 zfcXXAT?ZSE9&_?IaLH-DJHR`_J}ftUqxQVN`>b$kmWa~7grwcbevtUMbos7byM?|& zEeIYr8f_u-o5z{bj8d<+7Y*k zO1k-Uwkq$!!HxifuXEYkhU{d_X0}4%MR|heC$@Mw6*ZpQ?{Nxnp`bT=ONLj*qMqe7 zNg{&w@B3+loC%*T-bYktOZDxJzG!QWp|);+%_23 z4`$O2G*i$f%SjXR17Efhcl=H+H5=DrZUgn@`Fu(|+I!d9iiZB9-)v-N%0cDqt@r-s z+h-4YfD-a++(1tEp;#qoOk4{=z&|1<_YP$uV+C++otQUb!eo~X(l#?0+QkDKHMf$3 z=D1)&pgyUFcwkJ-*KJ?QEnS|zeFJ0ithBh|OC1wcyeF3G*UKUP>qtWT!J}cv|=b6n) ztvNdKI2*Fq(@BF(yZ~wCYdDzMNo-SLptkM0n5f%2=h}mV$zf~2@ar31InJ5*i6k3S zE2X-|;nnE*2Y%GErRMjfu#c^}RyK<9=EXd){e{fR5qjc*%0{2`5t}v zc-%Itb7wtuTjF9>++9_fSXa#`Cf32tUH$Y0^F7kx2h?*a=BH%uq$icF4z`ozn~9(I ziXOF!V)Os)#|MIZhSiF@-qiWtGJ`Ks@-dc-CQ>mu7voEEHF_G_BJLNx@2*sS!55h^ zl(u(}ef=`Kj}BS9I9VB~|E_<%8J()=ouVg@1dsjO=!Q;rW6`uVfV!;urkA~H8 zXUV8lh%!CX)XMvUeX;+ZpQpgE)n{}NrSukXtO`%y<0y7%?9*lIU(1Yj>KJ|?P3H_j zH%9ngv-L}<{hQ(~>p5RafA|tS&WdaRDtTkuP%Kg}lDcL{D#Df;$z7ehkpTgS+dzf8 zXP9Wa9tL;GOcSn_b1PWZA3%pao$q?3=1FXOg1c9oyhRTesTS|5tWkW4OgMDbbhE{T zj`PYI&zcm+-hjZz^n7W>ipR8(9km zYc4aPN-;{gb|I04qP8Wnh*t&C7PC)xbW%Kq&$;Vry$WuAP%$;1{k47A(V`ES9stf* z32dw~+IBQ1ykI)tws~mlM)$?0ki{Q<7J@WQNb%CgvO1hfU}Vmt^|H_R-x+F^r~UDe zr|~;is9t*jP;C~453YAqHhoQ?_d%G~bhwpj{%Rn@3MXoQsLUkDbHyj`~kG)9~+ z%k%@KOQE692fJxUlN~ATQJi5RrLKzZTe%j(5X~3pSFvGyLJ)RKu*Q*tBtwU*G| zjKXXQ|L($Kur5IEd9PwO6npPLVj+Qz^u2{op9ZGh$1ARlavN6>5iBjz^)CkJB)j9% zt3Z(a63VN%(Tz*}0;2!2n#&^n^#`NAqg5?-+*WMvWQSrryOp&*e0O?Ypsp4(>Sb>m zbvijM(mzb$%v$Q=*F;%gE&ZqaD1$ zQ*+Y3$6pZ_80>7Q8Gw*H4TUE^03B2%qH851&`t1LLJ=(k`QH<)GaagL zFICH&npS2MQItkHAjXHv?_lbzL_fD-UXAZ~`B5CV zAQ*!d9Y|pYB_Z!^J_uHc28F{ZFKzP4G(;06fDtkk&@$g^HrjLNk#qnXk^}7+Ie~&f ziAD9Byh9CI;^^FeR1&xQS8dY{Ymv!tf)FmUwhru`Qd1wCW=Zmo{HtQIz}VSN3|SAB zuwtAW=MF3o_D=!r#VvRhr9}mxF*f<uM zAnt}vG2P?3C#}m8VSzL&iaCAWG~yf=*+}gR?vJSH{crbC1uB;vY`%GQd6m+a252jv z16$g)LZA{zkZ^a?Ydz0Y_=%(K{&8B<_yu)X3wQY{vtCl9aiu=CjXm)utjZ&SIo4fr zLi%}hLgTaA(Q~nBc~3=mQ{Ea{+V2VNE7nX^L)#pDFFEwi@YWkCUBe`L#>=<7W7%%y z%#EhU9?eGKq(S){*#)7s=TmvCh%dbWa~GsNcfX!TWpKcTs(ZI>A~2RJIoo; z7G1kN6jQR^lKP$$YJ@FVZ$%Yu@BQU-QG@&w^4-m=!W7eYLq|WtHtFXC!+%kW)%}lN z?hL-Q@WexnvCLD|)kS;cRT(^trPcV;vQbc8HS3nqNeCTJa6Kp7Uqe?j^pSrzE*E1# zbD+YsA6^-KSuy9gc8o8d^GzB0SvFTq$KHr*sCL0_Wey@@Kcc_zaFQ;kBsTIW83bCA zJydpW#bCaDWQUvyxHwP{t>k^_G@Nv~z%r3!f#2}iT4}~bR^P?B_H|AaW`5ddzVr|X z8dk?I*Qu9x7pYnlk$i1!lU+wRfSXd9RW3&bq&Q0 zBdugjCS$ghSj(8W>ZANqbCjm(^0p24FtyX9F1FcFa+xO$_P&JIi$4#iLLJGxmFNz* zgo_bF;Cv(|66zxXL_;9643zy2eJiNGW%3C>bu9rEYPLEk_snT8!*_j1Iq$|Vw&h%G zoH|=E1vn&+Ej4Ii%^@@yI=XGEuc)y;9>nZFSwD|6nNx1`E>b$~+%*MQ0)h|ALcRK} z$_*kj?+!Bwi3P5)rNA5`L0S~St53ynGD(vIqBSD-$D?ZT$Ld5Q9)sQb$+h@JwJY4w z8`}B)V0xQ(Lw&}%(eEf7i9;W|3E=QZD+E14@sxG(Yv&H4l&@&GG&AH|^cN9n#!0G} zz&a${ws?!eLsTG{FDJTjYak#mi_$WyO(Vy;2yWc?;ol_FD_ECw5j1%Wh(zT(9+ zK$k+Rp0w%`+8;p#c+sSv%ucW-lygF{iWnK96zVp|)4+p?ar$iom1-g|4GRqs*S*Ih zNs*wk`yJgGWICIt!7e$`==5(q_pf3I&e>5IXyPz1OALD_Y*o#AB)HYeqUxqa0(sG zCfd~L6iF3@=>gAjO&_GtkF9TUxY#BLrl{pBj7~kEMtM!hX(7o)9f9H+f~y8SNBBQZ z=2U=55K`VY+Oo-sZp#z(B>UMQg)TB^cZJg83zNWPgT1Q4f+}NYs;SwOxz-%Wu3Q*6 zXK-%jip3(&ObRwRajTJ5Ar+mCKRBU_j_S-lxpwDpQ1y;`qUeHcOVMNXplh zfPHd$6jos1)Z`H=6iy`T@T_%TbE#83M+jioprcQ|xrKCDOb-dutWVENZ}Zqejkr;L zTE0x|FenjSk~v-L(%w!MDD*eya%c#b45OD?6!C@rt!jG&x@6yFS=r?$5WPqci2|il z-0ieaZD9-P*6rI%1zRqoT^B)`i@a`jQ;7%+p7PKm0486Qc*r^p32!OPNsgx8>}fH< zszJB0OWB}_jK@W3PkGc9BMrq!Q{BC&EBd7W7tL&#LSd#9`Uc;D3bjuw)g{mAl0hbQ z#VQ!8qImMv3^X64YXvzLOn`L^1w6oO=}lE0yb2lnf+lf}6X!tA(o--q!4HPA&k)Lb z1_!4!dnIojmEHP_Rno!`2kYp5z~}o>U>s3Ia_L+k+&5l~T4QPJG|p68u;XYS0M zxqrc#*)x02?00uR-w&*2fW51FnUodNEhgUq!(?I#er^kWUNrUMz@a>_n?G>zS-u;d zp#-^lhR4XpiCbbZVRZ0c&&CU!VyOklx9bHTS7|zFtbMDH&ks~eH~{su8y-yq(vbroWW5|Iz1#l*85S0N4*#u}EU`(=Kdkmu2+cUc~Z z$r=8D&)sN;M5BkmbaS*N z?gc^}y`zDQJOz>MDA;3RpI0?wV1)Q?qx>J&HeAdv=z z{CR@NIjZWsARr7<>?_)%noHj`$dbZfRj=R2uq7fV!1Aa(X)M&VM=ElSd((KVk6ZaV zBiI%L`XP)|O@r)PfDkbZ%XW>QF~YVK`X_{n<-!fdB{*R~YM$ofW zW49~>PN3>E-joCZSX!kv@8rrfyr8tYAZM<|i-kamb~BkaF+%l) zl7;7q%h*h;@Te7gNLSpJOaJ0xR&`HbV>kF zKv*|zT1S3l?f4Yw_}@PYk}hN0g|rTS)zR^wWeMl?1WY3Dagz{?5drcSN ztiZuPZ6k*~iyEB|CBCR>ZKFvVF1qAgDGS%Wx=XLyJzKo#8Gj@8YKu?qg{pQR2}}Ei zi()bIz?aI^MD67W;01~Nv;J_lA*pq$OcJ-49&}Ywx#H`Qb3}Ou@P8T zi0U-%jCp~%z^Od+Zv7nmNDiB0GkaP5B767@ht)k5lPdbYd}T$=?GPz8M%L84jRF{1 zrD59Otg-uN2&*n860Bd<$LPYdBaK&1D7Lp-{WPeKg~_DsYe-0p4I~L);g;4eOq5wm zAw}#GCcR0|3dEV6;@>XQM_S{68!Wf_Ire~9o8g&&BmAb#m9-aQDQdCm9%`-9lg=B` zp-8k0m!6&#?OXLA!FW9`F?RC-a$KJA&^lHz4|w77$Jk>X{UEK8;d|@CpVwnzvN|xQ z^ul(wuKYxN)*LdV=m5J>`sDig@EMvCrVQ*^)$iLxe0EN3)r&IRVa#ldg(;`R?f!S| z06)T9ZvAjA1$g(aBIN-vHV_p%v6A|nP1pIv#KYsnI*sotW=rWCJs5a0yLL=j_C2hu zc~U9{*>0Des4(!w@NMa}n~BQDzclx+oy^}?9gS;)eU0f(blynRJg^p?CpZj%fW(Dm z7aY}Nsuiu76a^+J-cCBH>H~T3^=G?vb${H&>^6RpXRdv#1^!^NvP!4fp=7Ir$%f)O zu}7jY@4xJ}W`4et{k@VQ70~&)Ah{NR(h8uoXoYOn7d1Z|3(YdBloFjoi6lILFPBE= z%2;9^QN`Y*JShKCRbpxifOr+JXdL{icsttt(c!@GZ?b;hWS)Mtlob@QZJb(82{^m5 z{G-2`xdKuB+NJegQ#+ev0Ke*W1bDqA=pfKG=Awud(;-G~R@KK>cuUjohm{pP^vxHC z`s4)uc`w;5-?PUO3Xc&Tzlv5IOOY+x&Wf|PJbvWBPv=N_T0HB(c%#mRqzILt`i?)X z{X77sNeYK`WAjsSN(dPznv!8_;xh)iqwh7!f(=;XC{@5B^=Y*?q@-&-vGg? z@V?=x&#C&_zw1?w_Z>XhfAr+Qc{^j1hw&=8I+v2MMt=tG|4}H6zhSgB_vs{8eCNnh zk#}`DLjTMnv`_fxeMQ+GTEn2nDsy&EAs?l8G`)peS?R@IIMH`2!gdfC^E`b-J>#~)G8q7D>JI-*SYD>8pm&Q>k#Wri2q&3^vU`}5b6 zy{DHC{G;=?!_6deZXLhoclyIw>5+=n54TRI-2U!kEphDT%Ho0TV}~*Xr(6^63=$7) zsD~}5U+GTUCH$=a@AaqE-`-38n*7&4@b8i1DFwg3hx_?lA+`t~_HUg~1R6O5mokZ8 z-BcYvgZy_6Upc}V%apK`jO5b7zy)fKCHk>nHKxmSo#TQ(eAS$(G&%IJ_Sa>t*;<>! z;qeQSjfAtrKPI<5Hc|DDzoNYblN0Nn7vpv&hF2KK?e}Sm-_UsQbtt3)e@3O0BO0<*%;x zYIp=m4?X|#X?eQB^ysG@>#f!FV@8*9%FVrbCfS9yVgWYWn@hJ#jgJ1i9=tXUKIm9~ z3cvr;q+ipeqkDfGzj{c#dgcs&FAZp6FAc6umwg#Jx?3|iTvYy8U}A<+X02bXwBwtj z?LKoOr~T-#stm>S_vdMv)vL9{KdQ}`vvS3Q80OB1w0g{kk5B0*&4i$Y5E^|fxpMJSF zKV2+~ajZ1_Ee%$)+g=P~19e)R6@Aw*k#eqmf@`#CTMwF`&Km2ESKXQkHG0tj6|H5n zE>D}E9V=7I9CllKP|lVaYycT;UgcKu1e!H%9pqL zJX>-BfbZze)-!56dcy4 z(nRSwy32f6p(`|Fq@-8hPH;lstJN@HDH>Uuxb=V-7WL(TfzL*1=evRyF1ktaBU1jO zo6|_mr)j5WS<-p54>xs(!l%8$67wV{m3X`y)mC3F7bL>{!bbh`0^s1?(;t?W(KsE|0&-f(BxIk-GlPTZ$TWqT-lp5^1&c_ewH`mkU>YF8V-?=Kc01)l^CP0xA011GQYnvWb4&=4( z__{MQZk|#|LP(RQzfE=<-@&lTeAAQoQ(Q!0n_8|#Yu`z%OZa!RmO%TQlK1|_Ny0&U zh$#Mc!2Q5cfT0<~kyGLcKoyJHLTIN9|DgV|qP5!eU4e;dEJqR09y=b_N2t-`jM_s% zJqrW#7BCjkItSl_Z9Kbsd@O79vGMYacCc3(L`U8uD-(cHGY-n`7~~-R%$gM7GXi9h z$ijuSNaVADL^chMEx8os_eB=dTl_!RCj&+MO6}{uX3(>y;PCyeKdg03HNnjdpxMKS`fWj(5t$wb66o%@@HCILLxp43I`K zWl?zgPpdv59|AG?1WAQ(U;Uhn^9P$_on@fB2YJVx*%6Fz8mikkhm$$&dfMxaNzD|> zC#Fg6Y+hAaq^u^ijU+AJN0GaG^`@IQ6VV&q=x37&&U1ep5;anMfD?}TiCuQT)dp+) z_k$h3UEJ_pZSBpl)=YwlLx4#JM{57I`lR{yr2YK-V*7Fqx z>RXn=$6T`7?`a&*22qOhdjyiia}-2&7b-vAjDfAdoHb@l8<>V}xcA_0qZ1g?ucF?i zDW7oKbZTq+v3q%!aDZ0eOln&_8_i5)PdaX`KB+ZX)(ZX&7o$=y-PUUMO|hOPOK&>) zZ6AS1b6NUvSlSsvE=U3}iMQp^53F$0g4bwu=s8gEI0?LAdn6+LS9_!j3sB`W0nwrX zgF_Ar->oDAt+>E=r3>YIFj-ni!VJQWH;lqYmky45{Y(>wjZs-#l3+>t#nS_WWtZAu zQC%!ZkYXFu5K94Q`*6em-t&T4aDbG@q;sxG??tjybA)Uun87)K5S}Ul<RSOtDOVqL@~{e0bzO3q$0WqwdEMkk zTx?u%WX5rq)QSF`u03iz!d)n0`OQCIBs1Q(Kp#7bpyus` zq3Fm#I=v3&UlF%I=#ADgh^f6;F_ zIUuhz0VFVyuk=yCi2E~)oH<^IWi+u#nis1d(WaTMMe_F?r@it8@H}0-a${(1cXSB% zpj9fqeR)lw8-zux-f1zad&6VkE_*HEwbAH~Hqb@*o2b_TxweeYXDBbE19VXGG4!l~ zQ_EB-wmnq^l@rxWs|Q; z*TilSD4MnnR4w=9@OZhVgGqM!H+Lu76z94HH_1i1+kYP7d={fF%08R=VZh$B)LG=o z_?b`2p-*TdH%2gi_p)?NlpLYxAu0KG+(8jZrk-m9(r0i~x`UIM$mFYJ%w@^2zX+GV z&xSt1Oi%)-xzT;O!1BQ?pQtf&_kb|~Qj9ygcdH&hNDpj(hGwCg-T#9=KOdQEC=!>& zkS$}zj09>W7t~Y7)$(S_n}hH6?1Q=zO$8*)+~^lMX1NE=Cg04P-95-F|d9NS#AfcxEW?vRvcKY04)O| zQpc1}1etHrJ68!NMI<3tlG%o;3N~lVX>>2AQM_KzO+qrX9R;|JXlRY0m!187IY2X( zd0O*fw>CSx5fHmh!%Ndi;M`<8I>y=L{nF~ zP$=0fUi!`=pJno_zI-Anus-+dGY^Q7wep_Ant|q9BaVh(ycG#VaI+S05bjL%>#raH8crW#q1uzSq3c|c03Hz&1TVg+=paoea|BvS`R69mx~7_#C9Kp6uTS;+=%lPoYG2@V}5KNdbQV)YUv ziyzU90fWj*fpKK*H2C(_DP=g>*j4+OJXr^g1<0}RaIz4FWZ?>QtK(YrMf+}(%ooIE z3Fq)!pb8f#6FGrc9z{yh6-4A^ak)A%W8mQfzC8d%9x--_9_}G1mmr&t10`%3#(7Nr z9dZ3UZ3*g#)~AvB%UtvAM*Pm0vKgac3ajZxv@PJyZoSdy7)3j+nqxu#ma8n2jhVUuF_?pL32*`S z310!J!Aa_z{fHu;vEX}69$cgu)0hf8GB4lr3h<-dr_jb3=3b-c3Owb&)Sr?Ni+QmZ z1yr_U8Y$k^SV9y1!{>Vd=2}eWSEB~*5^cscCPNtI&5?l#pbDHJlekalCCGSLUK>8D z*@qessFF?s?1p$cfl>CjO}}_C4M(2pY0_mV#dt>^F4v!j{5=_R24%u+!#^c7@6o=GpHRw2a_y{NR>WZ zxlN#o2GdwkOzSeiGF}_Qqv@oO4=?fAqOV7#d0h7qMmxO2mQx$vd__^C27j~WK2APEnt*w!Fb;2-esCPp#@e+1EN93=us)#5$zV> zt}w2sNQgn(h~f-U3L*kwjZFTp@9D1pGQ=ND)M+CJg4 zRF&lCp;>?`5y+*@uUwzz%IBDIbPQRY;B661f?`~Rg2|oX69%=~P^Vz?86BM^xN`8A z@?g$u0Tdf)e_E@2&;WVX9Uyp73bt9QdEn6969?}w{0NfuJv6+=nA{1LvG^_s!odqgW)hd39WP3 zmSmaIU}Qd{ERbL!IjI~oI}me%J)h?Xx$k$pLrQ9yk>$F7oOoo9%x2F-8tM@di@ z^%Sy*CqX~N9-RkLv_d#XF(*|W&zrDBL&Hgt+;SvC#||uJ2Uf;*{B`>DYT0?GGT#@+ z#knq`D6=$UfV9ACj=yWSgBHefagkvyamEbDyC`yl?%>i}7Ay}vPK zD`LkEr!Xv*FYB6-iG2^=j7JQ9sIgdPlvP$MDCWG!;3{^wX@*GI9wx0C3FZ+#i{<~a z4R+9%W3sXY{e2&jrVKD-zGI43H{ur%cg6>G&mmg?&c86q1 zyKvsZHD4aDx=+uS`WWF9$hm%X6Ni+kA6ecZtM{y_Z;q&M%}Mv{!x$#EIE*F!Wm;&E zZy3n<+R?QpYr65idoBZXG7JacaqADSc0|&!+;vejN9^P*p>|POIQFjV5?IqtF=b3u zBgZk=&H_7XRzfoCq3NO2=I74--T@oet{26r>)?*=gFiv&apAUenq3yq!_3-D)q+emH0W zC?sv8sd08fd0_nw=%sK02gC>eyQ~Nv@5u24=;OYONB}#3i>b_wiYsQa{ws96~@ zW9>_&oqh1?x*Z{EwvHQY58ly4#=m-2zqRO=i)aQJHq%n0JmuZvfx*8eI0O~_F%b>I z+zGCy1>vn2_?gm2Su|m^4d5w5L%8}I*wW1Ip74|*^kb2cFteAU<9(4&&t!g!c1&WV-eg07_}OOlv$msFWujNAI?K4x77HsIS{nbd<)A!* zWtxiXIrDn!{GSB=^x^gcrrV@WTawwb7`Fe@*I%R%Q*ZO-V{5qagGX}{O@HmZt|xQ7 zBW!`9SUJIzU+jO$6$QIUo~#*=i!r0omo>od(9ORoT#?^Abb;Y;D#_A4W1s9=B!E=1RglWYX2yW@CBcP(Qfap7B+8eI!5q5@N#*f zM=wUItqTAh95IL-3k~>tD)jT-!C>Fbj*^+V&>L!&F2>Mt`WMxu;{`Q(DP;2Xft-J0 zE%O7n2ii)E9_JmII3vuY{e1Jam}J45GakNf0Y6x&I%$bfEsWSGPZ*nYiVl8Y(--tXPxijC;fB|C}Wq*>kzgyDN?ZG&L6yP0<@aa-A zp|Vg3VoOu69AZO>`}Tm%1>F1i?`KLg_?cI=$Mi1D8@wJpAbeeh*DOztL#yI(jzJ*@4j6T9Uw0%WfIAz8M^}W4wz{KT7XA& zDauIqsh+SWCTQi~xIdm)5?E}EcEr6QOwOD#P5Y~mWl%l_Q^HUHzIVsQ&xp2pPR#N6 z2Yr!2dMQ!vZ)MMS^ZU!qYow0^UOoJ*|JWaygy5m%5A#<~${r29aq`!X?+OE67OfrC zQ8%JoM{dOrq4EDw52j&Cy0-FXvw+X0?%}*SjUUS!cBgHck-S@(&tzX29(i)Y;z-sR zKM`X_#UoGi%=gLcuj!n$8Lnbnf8u8fi_#0eW*^dNG!n3Dr}T-y zoQ1XaC02cU(Y0w6yNfG-mP`jB^@o2wf8G|PZOhoij*gvWOW-Ip%kw zP{*XA0BI|L|ke1#TOwl`e8P9dBg3pvqVV7#(7tNbu!ztOI zJ(EM4|AB$_HMsu9htgM0J8KbfpvF6sWm;eUKnrkGcpmb~a&k+}8Oile=dFf` z8khTU{zQd6|0Fcz^l`LI%+|D8?zK;B<5q^`yJjq2@n2$+9t<=_0;Wtcfu}-*d34Z{qn*;r7kf!>Yu50spp;qZFO?*g zjJiltm+4k$O$QKc8!&lRG-;G~9)WkW2g5DW2&ISW@w_bma-DG@bOF5eoDdd|1;Mid zpinV*Ng)kiUPWd?BmpjHQ7ayWA(NIpv6Lf2;m{G0p-_;BuKzFt;p+j*6v9TRw~-4dvI|^6h`W+L#GxhA^1vRECphAi$L!%)3eA}aMzA2KsvC{`;-CHD|9dsi*jEZwCePRSsuw@CS7U$J`X!-VWUP|U(2Fzmrf ziGzH4vs75e2py=@o2Y5N+T1j2a|L_SQ$c3@McfEbS^)U!%CFagzPtT>BmLP~1e!OO zzwV9|d(LHMEI$IuRN9AJ_$slYuU7dh<~{0BJ3y@$D0JID*W~GM`!HDA(4QzKobM#5 z5V%|!*J70C_28{81l=VApHtncC`nZo6tqE z%kuPA@AX%6LXcdFex21TS?tuy?C|1mm+=pCSDUzk-tnd4B7jD*sAYz`&s)6sMK>-$ zRo6N1{nGhHLkU2A6tNF^nf@iQejEfRA1$-!b+M@El{r!JA5zFm!zH zh&1FFAp}9vILCud9>ivW&U=Uj3^VS2W)sq&DPn38(GcgapZ$;=7peLNp%(9D6^Ni3 z$#W4{NOz3{IDyTPUjS%9v@;FQ+@+2Ja^~`mE}7A0(`0~PfTk2_j0W85$urOZfGau) zkFZ?fw#3NOzGKz%)$`_Hvx}%PI2ma*S1f`8sK@#TP5-8rlu-!{Sr0CgRt-QF zxwJR#D63oP*u%+6qsba?ho~-!}J@>hE z+T5sPy+E%qk%7G44HBVdT7*)SE%kGN-{T4*FdSUCRV(No&o#mZ5497N?^LNWENs3s znaAdM1sHz_Z(74k*YV^fese%ecrm+3E$g~2fJ8^J2;yS9{4J|47A{C^Pox&Wfjw!E z(M&rK0~RSm_(;l8LA|v^x{Pxt2=zeJ+20P>gA>y4H-W5gM(ZUI$d`)VJrm#Ex=U%$ z{-Wd^ zW#ZK(LCVrwoLAruKqdIQ9EAX1i0}5$(|5{J-21qVM|z-E?}cuvlvJ@E;wWTne({BS z$$a5aB2%8~PmhP`Z5`rjaBy<>9 zeY|Hn>N%8!^qh4wZLMtrb9>bP*!_~QVcCOE0Iz#kT=WMDTHnyugj+qzYl&S>ep)f`Ba6Wy&^^B4ef#mZd96WxLzN`UP6 zf^gl6bYM44n9>FSBy${8l(CQPa7~gV&1zV576Vktg;4eZhzIBISN+leYVk60T59Hg zo*bZpRp4V0-s8TyE>%P&-Tq)SKuSbi3ffcU4CGufKvTr-ZxWF!)RIEmCULMaSB5(B z@q`gsO1|={c$Y*T2n~pww*rlu7codPH1MZM0>qn^>xjbUDj ziYf58#DGEI`uhGit>wpMTfO1p7tGLNjjFl$YqCQA+k#GXA378P66X6{$OF0hjl8$EG$80#EHe=wMT+vmYtRz(Q zCDvV1T_J%eX+@MYnG@|NDWZt(0w_DL3l>9sU*4t1FKUtVY%1;t?N8+>&{-wsd>z%Uk73b+!!su-%O(6E6fwE;DX6@5;D{R;!9Xo+HJ;EZTF&oFR~t`u8U zWJU`-Oacw7furLv$S~o4f;gYTec~wuuRC)1-tNG_5%R@Jh^Jx z8=C6#Xn<^*s-SvIBqmN+4GXCzXl_wp53%U72ASYiz+>*wH6G$2MVhJ#hXYz3VntSX z==T({1p(YiS+38qu^|8~aI$SY$Vlvp%|rFQk{aYBLmT@2e{{U(TNze^F`6q{zbZdB z<2@aY%u`k9ZII((ZM@rkmauYJ;Cn1UPz(p(r>f8wrkK}T+4@c&^TE)&9W>jss~Hl6 zZvSYw!C?_$MFE{H`L?rTuUQx*Gh)_b8rp<7kHu|d9?h{Dj| z$hY#L54HKpi`D{_wSkdHce+_a@MBTm1vRV-~Zq9pLjDj9^3`DB#Q4_cB zQC_1Qy7tiRRtjLIdEjuEg56DspXesTnJ;7sKZz z4>RC0#tPj;3|G&5^^=OjVqj924~;C{H)1VdI%5vO%bUjqU*!y<0%c@hBTx2|hTJS| zJK>JBjlTT9MVVHDnhp(~K(}ZDiu($NWBv4SgVx;!MKt&uPdK>Xq4ut@@qZ)sl|@B? znI;$xJV{m76@Th+6k%qco}sFIB@db$zh|x)cc}q(hhe)6kwG?r#`FBDr_2NQdGM9P zPL3y`ALjg)FZHuXBJ>+s-43%l?(D|q$xb#X^o?L<3CxW4{gp|CO3qmC2&R>Sj5h6$ zr0-4amhT?_Ty+xK3WooC3?b0IIJkD=Px)hzu76PB-vbo$h?Vz^Ec z_P`Rm{*ABnWr)@9??K)%>vmhvUnJXR4bzh5TQ?`@R04c(UuL6AmiJWua&xga3lWUh z1IafvMLT8Xu|DG-KgcmPZ8P#okP)Kt=<V;h(K#v&9>(dk1{wPY*zJpO? zbQ{7A8yY~74ZptZk){r&y;Y%7VvUmb`x5r|L_zs(2OwGTfxLdmzXpg6!9ql{5RzTd zm3e2@$G+jP-p)D1Q#;eL3%lF^La6`~rKf5!+_HbFIeT)~<+FOxF_7QWdG@ikb~l5A@1M;%ei|Nw zoIRJ7<_OrP$;=V}Ik=Ph3|%LRP)?3oTf4dz0UF)9P!E7Ew?g~FVP^j;cw&X{1ppTg zP>rjdK@n^^1Mpbj^+^bv0|a2<|5V>F68kn@u6ttCB+>7y; zt}HqFG-qFPsHbO0_vhaWtFlyp7(-tGQuruF>vwFI?ba@NtyX$9tyt-u!Tdtrg{0g& z3+A=u`R7_d5CDX~Crq+(i4t&;t-yoqBE8BYqJyKshE|PxBIObj1sP9-4Jg72s|Pv9 zhJM%O=|^y3;K%8@1rN^UTmileaF04hHFlt>WAu6%5VI%eY}6R9e(BB(S#tMlXGO>4 zRf6EvU<+)L#$f%E@R(N8`LU`P%=2urJs?zIS_4a}twJtTHt&1hd?3*9#4|&eUm=;P zm8wGqbCoTZ(puB2b}28)Q-I?AZF0Um6|v{3P(;O(%97oVE#C%iqDNt|ww;%eM-W<&p!nxL~ZNSkrzyoND7gc2fJEr8~VJAjI z-7nvV>6s`*aHB7~#oq}gTLgpZzFxAlcu^;G+&4rg6v4QB`gx`8*(9qL^|fbpQBU-_ zFoVf->Xp%2RR{PZfgVbb`2o1pJ)P|#KU5ktR8cb&@(HrZ!{;qZR`M>;tJkX)uI){_ zW9wP`OIkFKE-n!4)!MUa%k?VPJFh6@$_;@8%4gE?Jjdq-Js?^{n8bD1fcU_b10t8( zmI|LR`U6f5T$|9GU%Fy&=rU@c+B4$TE=E`9QJrgs29u2=zkN?A1nH z?&Pg@#4urss*^Z+9}{}`UUT2z#N`D@!1eiPOZqN*szL@yV-3>ea_4B(uIZ_%(zY3t zUfGRshyxevYdC(e_TGz^_ugDr-}R?cQ*m}ZW9Sy5>Z76|%&+y%prMmndSAv};!EzL zd2Y~PUYT}lQ}DF5>2&c(!&ZN{`7Qj~)MB_*UI& zf5`4e^_=?O7V@WIg~Ja9Y9_8_TvM%km^id<^K8!Nlu^px7Z!DonIQ{j>lSifEfn4e z(cFD!WWUmM#m=3}+s~dcy46GG1MWRNQa5jKcK%$*)6a~OQz6X(qR(#Ey}bMC<$a}B z_mo~vS8Y=GbZ3Ur z_}>!`8f@p}UbpWHSZukz_bEfVd|CtkhK zy!k?>-au?m_p9BVk@Z~1*DFVr-zSH@7drjs`0Mv)lvkOdt7l&?#MZACDu4KJf;CX31+W?_9ZX z|NQ?|Jxf>m+WXocUHDJ+bh+B`pX#X<)p51+>81apdOo{C8bV%Y_FsH4@Sp0rZ=gFQ z%m35btGfc#^A-&(tn512vpB{UdLFFz5iPO#}|VKrS9CjHt_ikmnp2`c75>6TT|_Sckv2C-`>wP z?K|&wWBB{}!vCtCKR@%va``F;{(q~UaIH2TL&Rd0$NXQ_(>G=@6BFI`e^gIw7H)~n zyEm_xjcaXN%F*v%T?%f7H_sE(Z^XRKw~9jt5g#nCzAdy)@&QH@WMY?#dnK}$OI+jH z4wSk(=q!~YFy;@@?f_GLU~A*PBiVW{?@z7nARZ`F{!B1!fNVd)%o3g_qmsQ zKw&!X>ofW-n+YB-=iYNz*(rVTkGtZ=?s4@KB9^1U~#^l^{3)WKun6Gy@_ zsF5eR4bR4)_aou{R#lqt#+2BX8wL5Fv}&e_J}BeJ=_;Y*o0j|7N&bneX5YN5gIctM zYfe5u_gN zukTW?+*TCWaqp+0gN)O!Diru>45V%r|Jyrc+nG+w9cl2=y$V6c z-J>u36aQFs_wg^sf^?-{PsY_B@2q=BV0Y{-zALvrE&2F*LD|?ki~K2B*lhZn`uufO zn4-@a!yjs|0AqW7OFy3bB_?V(r(w1WL#@Lf(j}Q#nXDhTurUopg z81AW&VwAEUc{zN9ZC14Bwz><7f7X+bgO*U6{#8RZZ0+03hInl0!6(ZcQC!u?V|Np0 zvQbgw;uLI7$cH6@i=Zji#sjy12mT95Vir+0RgXT-mfQ6RFN2L_zlbHAJHbZV>P8oU zJp$yVFN_jW6O!e6uy-GAsmn`> zqyat`Gt=iJwes?dwy8}^RAhR&XIy4EDok_BB=T;^@2?eHkyiX!=b3Q%ZvnjdH0!K{{0_{m(D{LSN2Ay?x$(J&pl|iZ1v%r+fn&4)soU$^m;&5 zMnP7ivsrVuejo}T-c<#@>}=b_m}C7@Z*Vcuf!@TqTE#J*`Mr^9`#;u4H0s_jNK3&@$FA8aM>pCuM7Gb` zIeSILDHXo{)+)|i^Q{|nzhnq8joTgJCk%Q>pJq4uh5=NbddBY07fOE%qRdq$KeXuO(N>#0+<36zK45` z=srXx-_xi;mGOaB-6X&3lH`OJkQ@D~X}jOL{fYJ;*ZP5#3L90f#0DNNX$J3Z8BqfH z`&6X)_OC2E`CiE_6003Kv-z@%lyTCtX}_m~m*!MFxT-`9&jC@bnCTNi5;J9sm)~V- z?f39{W>Gz$?kcC3^znozSFpV+T*px26Tzy31J?T$C52nLnaH3@SSlz!dcR#z6JGTU z13#`cg$+dFXix^m{-AgnFQFa zi(7n^q+P(NVOx|sY9u42)d|*$M_TdH8<2!CTn~r#b4|7a$sR>jTDSbNfZ z-y0TH6?Sm+!?+Fit;ZnI!>HRyXki0pf*%>Y83j_{Q5U9)(~4p~_K2%nU8HpCA1Cd;EXj!=fOmIwR({}>rw7B{cR4$xVsS%-fq}VB(^+;)OT+^S^BC<y0&vDDUa`C*L?!K)=Ae7k2tMy5ZZY?A!ZIEVvr>k8Du zE{UB)jf-utm}i2CeUs7&Bx!1v3DpL6k`26i^df3cUSqz8#brJgk_fuG?y0KD{^imz zNQt;znu@e~^tAo8<#uTeys0SdGPIX3X$BG>>UT;d9Vv})oRm2Ro%BrjeiCu}%mv42 zoY%2|@6k#VXC7w-X=y>b@z4nig@chN?!NWy*Ori?n<#BMBYRE}p+4D+ARO4FHkB?} zd^%s+dh_^U&*<=PeiVb%{n`sBrZlY&mYwc=chbX;kr%Ze$w30PH3OGJntCXUEezXA zR<$`w_$g2zIf5SpJOqX%=aeQDA%OFWSm7On7=fe2b_dC&f-ELM_pTl-D2owI^UT!p zaA)E0XPJ?cvDH4F+A$|KA@(Y`1M3&8i08uk$UuQ=Nz;D9UepYR79TILaM!d4vghEb z{Ch>LLG!S>!p_@hqWiiR+l$q79mIW2he9O6(_c`G&ujZJLA z9<85)*K)IMrl5tJ$#q=l{bm3HQjx`l%rpS(sE7^%vUxMIkU&k9R7g!1jFG1tHt~j^ z;KI4k{DNiFW@Ib21}Ysh!j>CafE#+{HYX?CSlur|h1TGao0D{j7QkCeXhu;vcgPb> z01NC?0Xgik>6*YNAkir3(OBesjJ6#OF6c~2HKScKK6)yvQvV4sfr@D6APWglm~7Rx zpYXIhDM0D?a$bb1Ktzy%>&*f5sPII0RvI_^{gk364e7s`pUi|75{}l+A!=!G8z-Q5 zPnG|bDp8w!7b^Eo(yesXg;-^;1Hh=5lU?j*XOZP)!_M5=-y~#IT!N$8Sv$|+JuS{ zFM$l~4*NEFxU&VI$%*giLAU~h(wpkerh1A{tNpK3re=qIZ;SFT+S`hSv$Kx}q{8(1 zvPlGc)9DPhG$B%rLCuYhWK`Dt?(L4hb|d)TpbM0-bl1y}G;M5=XatyFvh;L^Dh z@Dk3c!&RDX(%_U|$p@;!sDDnEuQ*r~Ptn zgu>XH$*nz|hK_;oiSUz~m;L789awn6RIEpl)z{WCJ1pXB3(^nO(IEh4^)xS(+8_&2 zP>)ngG8e=q91Us({IrI=S#V0^A{0_j+?9yW=cR%(FOVgz?NEDbvGDHL@XY6kj%;{$ z)nO%COC74~Ms}DEu4S_y0bDCyYdI{Ci4RemBP=^M7wFjxSOpDS{KU1pKZ|JLvx{0b zT5cuFhBninO|sC|bVTkv{E)k*b6dp8t3nlA_@30EQ(oP$%NME1xw|I??SxQdzo2^q zm{wF~N4+Xn(`YRZHb%h_mX|WP2sb&f2epbf-}$}*7sX}uthb(DVSAvULH;n~Ea+tb;Cu8glMrrEx_5E^|$g@RG*^dDjT7w^@!EKw^%FFiX=t<)@H28uA75YYmuENET06(;#aZPOV8&dSZgiIM zFL4j|k@BWN2dS(0pRbQX0Bmo z7;Q7hSj;^~wF?u;9i^JH5kl&tJ{`L-X9!8soJmNM==yAPAN7$gU7toO`S_HMPha(^ z_S^42*xv8$^?E*Eujlje^nu2|8-PlIANvu}J>c6E%^h_h8gMDA2ja1Ma|%S0Tmc(n z6)FB8557XkhlTTDNyCG$TW>Xo^c;pIn$rPgk;d^?*k9*+JBXx^gM8cj{I^{Dlx0oQ68T^ z3}MY68u)#c`_EBtDpCg}pHG1}&k>>g2N~-T4br0~(zIOQ#-#g%5d{&alQ?>@cjrH? zM_)}fOD`Q9htEyJNYlMPPYOR4Ku&1h`{w}&qy+OlJ&d>r;i}DToCYlfJoUZmB^E}{ zf7}cpzO>iA@PZXckxAp8wE$xG%V5OzDNG`MuHFz$<2HOfW zP7b*GCF~kMpX>|9KQvEyiR`9rrHL(0U3=3b_WA$?t#Hb2n}%6hC0$r55dbg9rUhG7 zaILU+4&aog3?>!S{I8TH>l*9_SMu9G8lRSA{)v>$7aN3j=kIuRDl%=0CQvIhq33ID zzF!Y0*Ddm%dFb0`Y8C*J>9i1o62>(Kzn+lneaT-)IXDY=;So2J z^E;aT7p66HPH~d>$g$fm3%&2(ect7BP%~?lGxZcfEEjscxFsPfUk-Hom><9G<8$qO z)w5ub1d>5i<~5*dwKgPb%OK!W2S3`>-0D{DSjKKWdoVU%v}F-)-{Id<3bT?y!=-nV zZzGBU_={+G!RlMm(X@~exY6Gqn^iNntT7th%F@HGaC_6^2~-|-8QzY68hx7TZi7Hu zyw~9?{))EYKkoubA&>1nw;e$s1`DH1fs7)^?ghB+kEW*B3(aR&pK3?XrRAb_MsAzw z4p5c+kKD`w9u6kkz+S#}*$U+HN1(ui;e&mjCt)`|E)5=q#a|RTDK&FYeB~NqEH_r; z>O#-yCP@cY?Z^(#ET`9bcKlN~bd^yzF zZ9)f()9=(bp8?MAEAicRiGry8aMtg*&~L|_Js$=sgWFGnA3r-28WJs9sxTZ%to1v~ zrqt>*1Ig6S%T~vho(nFj9Q6b8Xw#$5zntmA{PDTWR@3LFm1e{1yC?D{YYti-9UA^x zdgYH%m=&U!?yV1_m6WR@!6>AfY`o8xVI#JoU3se~)b)kPrcAplt1o>eI?lrbVw<5q zPN+7!ruM}*ngxeqbO+HkqS50lzc|4hL$13u!=5zOK4*5DN|L!SyXpZeY=>6!ZWylJJ$3k7-SJIHZ6jwxIq$wqEzCytKTVaDhL%|V z?Z9v-oVNo%r;iM1V=5Ncfp;n5ttHM8%S}i$w#iWb6cP;bYH-ofEV4-DBg0k^x+Elm zN3_ZRu|0mHL4Zj-0ahLYE6IO8kf_mbTRW};`;{dyWJGb>L7E9XnR7%F{_k@}4l!Fn z{|!V-ghnJ#Vh_lZurT>j@%eqn@%C&{58U8pVHiP%0eK5=98`_%(*p=E}Nr5PK zG!SgJYO$(_U^WMA#uI(4F}AX20XIT1v}VB9-mi zWN@+TcpEPf9cJ*qmR@+ivZd72yFBq$zGG6Oex$?cD1&9)!b@3OdCTMr4K4rLUV_K3 zvW@O~&WpXgmfo-@FT1Yr0Ze4Jgktm?aiLOhY7#HUf_cITIq=_~Cqo_P*+z|IHGp%g zhhuQ-8Zv?pV=pZ2A8NgAbZH^K>3czGXoLc3!x(#=82r#Vk%vBisi0lVs(XB7dzfzF zd?SI>en?_}CorRY)A#B)hGQd^yvs)O4$RN)ph`V>a^s%be6hH)UE^LqG zoO@oGE%__=?FMcBknQG>8;^0Xt^A~tLsTZz2f;+;^F>=M;w7Q9Ld)i8^DTbh;t{L& zD7$--y_g|X7cO2{30fD{#%;@=>$E{CtHz!;RoDUx2F-ZOi@6?6&cix>Dk{t7fzSxI zYVB?Mjf*&0)iQ+JyE%6i+7sVXz4l`zmG!$(nAex;o?1upaKiYX4g1j^Ig8O)aw%PcnGu8Rf$reioiYP2H;`x4%7LXFjJMg*e@}y>|ABAzcYZfrd>S?=Z?*L25D1UWgHbkfn0q&0JA{}sQR`xocR0@@ zw@z2?Z{Sr%#a{zwn)n%b{ik8Qo5{E5w^I2sP zwoFZ8P8zXgCM^2vmrC0sMTeUCzP|3+g&wEh*kAV++r?fB2}39foFM_X(y}*iUApUh zN(SokQuL2^zFX%#GrQBQ#d0V%KIo{vx|qp$i|X6}0~LybnoI!?+a0WQd*j7;a0P|& z9AuyK@Y%J*y!+)oV#`~Xn-$;cL?dMxcObD^o?IuFgEX z@I@**IuwXnUDd0m1*2G8m*9I>HdtGQgS_Up3NA#CT#Ff)ND&qzHicni6sPz_b1S|P zUb3}ju=YqURQI4_o>s^CKL*!)4W^F#*@aHD z;OY@{8_a|@P1`_xXDLLLi-qA1q9AGR%*}u3PG7Oj>@h;A&su=Wts$FlggRQC3b#F% zik1<5@Y{yNmCO`%Cg4MR=a{vZZqglh`)4lsu2UNfTtz4~t*3eOgL z9@@P?i&?bZnLxX|>phqc3&S&ZdV%*{J$wbSy)S zpU#ng1(c5VAcnJiEn}tRoQx?=N;?JFu5R9XS{cE8p@j+dzPCJ4+ zCw+cLEQdj-=c4+GV+D=@^TZT1S6p;><;0$h7PPZpxpw(Hs>YENE>YhYP~$Y%Jg9Me zk0=%V<*HS#3d-X`D}X0|iGn~QAJyz=_cmaw)4HeU~ycH#J&7yhJe2A>~&wPX^V<`2iGJ8&zM z%Z0iq=(c8~|8cR(0Yqkk7Ip$C@`T{pH#`Ie%(W;$N@G)YEqO2WjI9$`%aA-P>Azp4 z2+a%Kt|h?>W$pT-rzS7}BbAaJ*ChhkrklBitW`Soa^b!T`Ojj6%oO?IQFv@UPp_^)}`d-%{uverNJo014yYbPjjx9B>PMaOPrr z@yIyv=9SjokC*4O^k+X6rcMA=oKoxXibFSwn^Ou?~>PT%zzes;V5VMOhV_lOe$~AcIh- zIRc1rHr_|jj0-=zGmR?Yf+B4p7=)?ogIlRF6Rr3{okNuds~n1&NCSeoCNMnE zK6NA2Q#(hn=`YEN^-Q0j(5lcSv;xaMDMgvfZWwW3t|%IOoTmNY=6k#g%C5tDP8}P7 zeB-Jm+o6^`Z~zBQDQC=X_VK9|PRhU@TtV*{KY6y{QWNktyLQvxHU00($xBd2^&y6+z`sg5QVquM**oIa8A@J0b3WHP<5T&^lT6kxKx(@skk-}!v*+(_t6N91e`u!SO64_WE$o3ps1$? zH=m$B&R_1CcK`3=37Sj`Cae7y$9@DUiK{|h1hv}!4ZMiG!2Nh9b&y#6?vn5ohFB}C zGdvO6gM~|lXip?Hvu~x?4i0SSJk)QyiVRQ&)GTS`;|X$(LFpW42(4~g5%|+YAI=zj zD?Eup{&ASH9ySL-Q(Ocze}hAJF3qCGved3LE>B$2)ac*Z^G9E}{N?HiA#?I#T{kEz zuWP8dQ+8^;7*zFp-*GxL9IAPt7ToCEr`kA0{26jhyT^fXNW$@g%+G8HDh%7=JE6_1GcLMQsl0wybXN-mjSU01&5+nh zgTZ)&Dp%~uIPvn3;hB3ufE;u;hEt9m-V43z%hL%pyBe5MtUD(fkUHcSp8Q{c(E#G6 zfeaEfRH%P++PlKy*&!W|%)G^BKxjF3i)he;JnR0WD8a1wQfA?!UhCzKGqIX?o(w># znKhi9mqsg^$%Wv^=nMHg@u!9BEa~-qkf~EZc+fKr<69*eFFbIO*wJIqB)jfZ4kgB* z#;EJuuV{Mkpxkd&X29kfhs;#l<|@WGmf7_>WwEjYLY=CMXV!A`uHD=0j^~d^z|p*# z`~hLjyK&!gg9v1x_CP?6lVrWJg;y64!LgTH3LH(qnpW%Qc#TiCKlYOrVt#7+08r*) zU>jE$h`K$R;(rDOS|9VL8BaSr9>kCdfpQ^>YU~A+kI_YbtJ_k1kT{vhymlzS?vE80 z(nni}1AbtlUbyAUG{yvMEqgY;g2JfXBneobD)bw-7jZ0M@nFC4bzVs&356Ajje=XQ z-YvCPH?+?zQ7=byV1CR;Wxdd3@tA`XO<1WoQ1*;~cYGDC1r9aCP%q~kK7q}J-n;># zNWf{+e8TD-)ten+wNh)IOZX+cymYAV77D87VJS2$H%36_3T=TBdijPoMTja{M+WSi zZaXkHCwQJ-XU_#&^G)iVF80@!x26ta0G$$qlYn1*Dn$6YY?VUoxkAGDt@&Qfj?9hLN+I%d1dIpujgYleK~aTZ zD($&XwS`Uk8N>2wvIO*c0RzLHC^R^ur7y;EK_2SCSxzPT){q=iFp<7(tp1KlcTMl4 z3}jg=w3UcD@L*ys8qC)^DXEx}d4UJ&i!|;1+C!MLxAuKJahg70RujH6n|@T@cVbSf z*z<%omv3SSHI5cIqfV;y+ee;&FiJ3kmiP5r0c#cNJ|OgtcG~j7BImnn60-4Ee1mm61^U{nSrS@g7SZvXQ<$}WRX!cXy9X|O;+2m{5CfqsV;^JmqA`e zIvk3B$-;t3N>No$s89Lr@`Yku<<#s`8^-YokF_CC=uF#T~iO zXhuNGye&#IgV98xOZMK~2{U}n+Y*7g^Qk=8ul98P4V0n&8qL}SjG>DBYhRE-=Qj|1 zSLvGG@201VCNJK4JK3Qvpj8uW1z0R(D;3g<1xKu&!%87O^gKUXo$d4WPBI~`&Z-|@ zF%u&&Lm32Fc8!le_v3<>@@RWBr)Zh5&N~0dyrQ9bq4~_j`m=-e!0t-hhePd%g6FBD z!+nq20X83Y;O`*UrI+Rsh-r^}qvs75YF;oFHr;kk)7=AV3JX}>)~{Y5-$OMvd!8vY ziX6u|--d?ET)V^~?JS+j!<*aIBv?+lle2^B8eK1~Zhb*xlP(o%{N&`|IQpr`hCxMr zLK`?AhEnJO%7K#MMubg_;J`PHh)h@np2(P~F)BwgLSRoEDqGeHf~hd)>yT&42Z4B4 zu2)_t?{R9O5rXE2P05gBxa=>K?gw z@BXsG#1o6Qe~}Wk(GA6bw!tZ>>Y@~&@=lWU%`j`c;PzT06m)$n-@F`}OU!y(aq;Lv zS#1{XtTfs{RUlp{?&pc=xu(2Q)BX^v86r4XGcn^LP-)#9j2&5Bgo0EU2jxW=DdolK zGn|8M!67h}Zg;H-?T$rWnBVhMO}tbMg#M#RToq4!cVWyyrhQ!BdgHb#P4DyguDkzn z`)uuHC-7dI^w^Ew#ORSNqg*+f^pE1vAC=jEb64v}`;~}j{(0w)*_!_*f2Aygf9=1( zPe}f@&N})0nDXh{1~85;e(X@~w?vRgZ=b~u1dsnlvlaQH{$V27kUc(47@>bAES#G6 zdYAfiSW3%n3#gQ+zPu541=r+}inn|q7optGm1{+2!op}zv2}irmQK@R^jeP+kWzsk z6~-_Yzt3D9r7QiHba1)i2Xj--<0~H5zIg1_seSjCH;d{}o9r&=UR{$I43xal7imp+ zH^0uB>9^Pu)AmO0I`E%=KSFvaL3pOm0y}-~eM6`9-Ohbas$HVLVeZv*C-Bejhmqux zY+b#lzwhtsSo^(;;yfF_;t6HkLmc#fps)EnblmCLd&NbRZXB^~HD&7M-+IrBi+*5Y z+}`~K4YoFVa&mQ75j3%Awdjba7#aKGj79Upk6(Kn#mQ}}*I}OfEuI$|a0LeVzf!;3 z)xZlW9TiWBrmFN;j=Bsgc!e*C^PRl=WIjQXf3>t-Hn00r=US##YlhPogZlVcgsD{U z;OP;^svqb2OZlnJEt~cgUN*w;tp7ZLX`JbI4LEueacO&_wUVd)@&aDH0q?@*`#q8q zp*Pay5Bph$3lPb1uy?2Aa+64Q9E}MJ8845^JC1bjk=(D!{SwRgQ1qKc!WO0xf8Mo)p>fUqVm`$#gz@IO{nljjZ#~wK zbKW=X&}%)*cMU1dIlklN?sMgVMpvYAV#~gc2*W0dj;C|+3y90FA$>^eqY`xQ-Hj8g zOJ!JO!N!%4c#W;kJhBRE%&xRCF)qyoVr`eBnrDl=*)P{lfOb&wl4}yp?|jj-IK|7G ziT-=OhM|plSbz?JnpE9SMt1?bcWCMO?ttXxDwxUWCZjO+gZ0-8{Tk0gw=r)!$pC!i zxbeM+0eMasqx%r{!6sNb?XM7%D!K*ZYV`vKOyTDG)IYAG#j-lm{cDRTDmCD=C@e$u zt+hIFgKbiXeQ@EkkQ6n!Uz;v`0$@vH84ZWOZ}51yE&XU!Tj=)}$!*npA4S?UY`uQL zN=myt@%G&Ahpla)#eVNzH`+ulqv`@;te&){>zcl+i<}Q={XN^aE~GAcIBO@|J9fb@ zC|X_1RMxiTTVw#%Xyi=m7BVW&4C5BUv@Gb9RfZ%@ZU*T#4>-;AdLJCeT}>5;MJX`%RFtqct_LjFgu5ZZe{B`uuc8H_8|HSK9;UM)C)fjtP^;_2hd`OAmi(d}ncWsVT3i z$Yp}^_U3~08K%}{HlW~1Z}(y0Lv+})wnL}zp_%t2cuZl9)8j2ysttp^Oup%XLSttU zn55W+05pYczn2+a;+(g8nOt~spfdaDy}KD4;TSS(A8y}uy|J#O==SvKzB1UhcS$v8 zKAb6kTlMgUL-k3w3D~-`nOQdTXLekmo3tUE^In!JS7%D3Hv&S@{PM8;2&TsJzcp6( z9B%&em3;Qlmxnu!AAKIG?w>c@0ow30eK*GR_L=hdbsqmdv|so0(c6T5d!IzQ$rL6f zJ*qptzwcx1JK7XSxHf}^-jnz#1qhwg2ixwhB5?d5-}m?Rbyk(@8}NVs{(kSeugPD! zw-o9o_oc4;W+A!s^Jt2*cCEq^>U1M1PBWaiB6~WQpJ#~c$?2;@7^?@Vley!^wvg`v$GMN+n8rDcvRK zolc#sVl%tbD@yWHn~W|!z9iBDo$ti zB|mQ~ypx`N<;-6{tZ75!yI~G4r|;ynt?yd$O%ZNAB|T|XTduCloxRfV^`_0n&lmnY z`e}dVa$<|?DU*rg-IBBaupfKxF!|G=tmB;SQCqg9-MzxFz9#vPw>QD50lC3jPcQi_ zwQ7IADwWSRS#Kb34)0KREJxI4^%?kmz=ou|S2oiV&c9}Uzh}4J=KT7f&|nU-{|m<<@gMcw4X45j&fvEoZgYW&VPx z9ChugHtWXEhg91=9)uebJ{-0T)xX}_l#x@Hb?3>A_ut2Z>^=rPS=0Uwc-1{3F?Ken_w*G*yv2cY<6Ksf_=;ZA#t)Q zAR_#~h?6vh;>2`2I21mjm&_jtARvZ#0f@>4cHdjPvPn4pXmY{=-q%jAodDoft>)S)_mL1B z`Jcqw9@XEe{e&s@TOLs;K-rVrl4iSJaJWKqh*Zt4ASByxfNVo93&2- zdb!MXFTKt0wlgu;s4#uI@#m$3#ex44NvQ{Ngzpb>Zx+P?Izg8Uogbra?C9VNS~=XV zUq(UN%Hj2i3~ict+fB2$uq?Cf)770)!$0_W)YYDol@u{-4|B_MQ2+3yo)}a&KaUb5 z5YbM&sdjiVf%xI_r-!{#{~8z1tP^)qpABwo#^woGI}-^$VBO4J9H3vOwSmNiU!Gw} z3c}YqW;cjFA#t{vv-0Ac%?FG{=z6#J8L=@3{jp~@zkYdzu z>hj`lFJz)<=<8NPpSU~#@QtY?2V71N6i2Y}Q0@1dA)|aGDTuCDNe9EZLjaAl0LyZ3 zH9kf8cocIj;0f|{c4ndxtUVsL9h=zq)l#2Yt3co3!55x(-v1f)4w}gmqrg;-#G}R% zIUZM&ci#x-wBz#hv0;;eMSYl3X*?(k(%GZ=;{6ionnfgE*Fs$Zs^q1I<5S*9&u_Vc zT-5Jf_4SDeO@`a^{YsS(qITm!HcCzjFQwg>dIX$Xp`+SzjUYy!ZK?^gHm3&i9G4rB z<)8Dxdr;7%RHoQZ${5T=iS=u4G|+wnFt1fSxbmoAc{cIiLCK_b#v9(pw}*#iC+H%; zPnH(G@JXHW`?{mv@*;;`^hbqKm}qWEaybv$aY?3SQBh2RUV;EKL*A~Td>gJM+N>5h z_0Pq=fV5=$L_$E~Va*rR>?2E%TNMuml5r~&IW6Dk5 zoSh~TxKo;SUY(jQ9Sn|fgC^(t2ju{uX}Qa)DO*zgcMkn>tHPNg(#xbLxyqoqVD}af zkO+2P0%>po!4E!#D}+(_f^QFoyQJ(cOggm&Tq|^smE$j*%PtxDnicCt1&tp)k#)9K z=P6#wg&Sj8o<72~8C87@e%pm_;DvhrNP6{ITPQ>VKu~fw88e#(CHy;fmZbl`*1Cfu z<7p+;hs9#>D#IO5sNVu|B0(TdI}Z7rG<#wknqE101a`ij;odG}BgBA&?%cz2R|A7( zB6p<#DDhp*Z756A!jV`Wt}NFQmUGbAlZ!9uA&+W~Y9?37X(bB0Z;S?tS=<6v% z0Y%D+G#OEwg$2}-7hWTXAFpmHU6nflLT9_iGnb_iNW<)UKB@<(y9cPJ#l~Yb;{Vae z;)v~d5DdWaUxMpYoo;`J8QTdlM97S1E9?mV`<2|Xkh+f3LMMdCk@vc!WTeYYQ*;&! zQSXvLg7LGy83>Rr2MU4mwWw3H=Zdc|(=+#GR3U(luZhlzioKnPVb;@+#AqF#$_iFPI4RYb)nAC0lmA^n*q!R-yn+j-BQ5Wo)9 zX7uyeYDt?`&eYKrQ)5N#IeB($ma7tU+=uDRhTzr$M&}}7PyjDc6NPkE?d!ykn{$4^ zv$?PrbMWIzp#EOL?DV~>5rDoV;V3)qN#vMIc5^%mbeOhB7{>6ZW#apH{B{6py3_nh zBSLotn7X%g+^ucdHwKuC6wc3yzUrr2sK#zJ`sLKbIAd7}+aGC-3n>qPuLM|Ve*y?6 zaEoAqGJ$;qF!(g!I4sn|%AV`M;F)y8H5pvY4d5!Ze?Ts_)&%S4H8#LVS0jU4uWmQg|1Pv!1AD8TMr@-UB3UDc>4sf!^8usnMm0;4 zJk{3ZI5?(P1%+e^9Aia9ioZfg_uml<8baH-vELo-cQBc%gity(4_S%gRN!6i&jC1{5@ACFK$tcw7fM3%=A zg(#tOpa3}sf`O(0GR93LR9~i`Wu`lGwOPte_uNiu&iNWm09z#@m#hRYzC9L3KmJ{) zmRubr>)+JDs#eAJ5{RdS)>qJ1Gmh4Be%Oq(AN zpW(YRNX5Rtl9@a1mPeupyq~cy zQvDhn^>d!j^n&=)7afN)JQjte85)aGEJ82?VpdDR4w;NIS4<(J>c_9_T-yafZpA>b za!1KNw*k2$N;F>f3a}M8;S~Uft{y`fx_tH{zm z%}0KjqjDi`6o{!F%T(<2-$$mgBg?veQK>G)Rx&B=GH8&DjsZJxMV4YYx)`j%=8LX^ z94SLC+9FG}pffB6^}U38De%#(b!fh(E1;dIpa$U`#Eg!*JZKIb0{y#Z7Zh&0?Cb;T z3S(huBmqTj7BHOwMa9Iz)r~IhWEmQ`OwSyqHkgViJ8A1Z@^& znto0gtzM0(J|Ji&T?@-6T>`uDz;)LIC*rsSpCP+khAvtiw1G)CeBnX#`h9|v{(L<^ zgx?FMDut#z#U}LU-i;#L&J@Eli>6Z~2Ooy67)-TgHP_LhEs9NF7`jJ54y3(cZkL0` zMfmiqP3Y&;HK0qEWyr3L$&n$n0VAIwQawJF;nHVj2%*Z|)|fy&$a`F|{EH>(?OsPb zOQn(TnG8Adtgie}xbasv4Jh2Gs{SKC4qB!eF`)n@5u(RD8CebJ%<#3p{Qk#hfTJ&E z(`dTf1J~`F!AF?xnF1i1&r;(U_yVw+-=%`0-wzwj{kCmdLUm>Rbdn(l(hg^C=1}?6 z&b}M<42;FtrWJ;}xLMI~1b2|nYz95l?r>awC~70C<85V#n}ou8 z`s-APXTi~}Y2KyXl*1#ie!%@~(J)~)V!LowD({Js_;bCj`O2SQig3kEZBTU=Jt zxN5q2=>C}zbi`2b1H5U8h<4IpobTpg#xmg$fI;TWNIoT&Rr4;&zg9uRml-KelVwAe zh(3IYhyaMeu+m<)un-2QBDFV~N=xW8)I#cBbh8Yb9}(*JCJLf^l)GU_H2MglIi4l7 zwxcETsJ#qv&{P9SIRmhu+`n$;uCC(IR~=#QE&A1O1O?#68n*yGh$UdEZu%%!^;F5w z*!c(ORe)gFLq%C9j&r3G)KI%|_gZxrZq>+EZQ*gba7V7YMN6*qU5qIN>leQY3VHz7 z_59q;ocE2f(%bc`m7EFPMnNA{ZPa2#-0$tM^${aI&I980Q*V4Y-=TAMx=^o(r5XIg zUYzMl7!OeUk7ulBgLof%nb~msMxB!~`Xc8Y`d8$&W9Gjf{O8%=Z~yycekAwftBF${ z$1RQ?{`c=EzwdGCHiQsgS&(U;x6W*0KV75SJP9~MLuXo=g{QKXYO;Gml+ZY_zEG(9 zD|EbC*NZD>g18g;ny{X_U~jD}W+~n%Zxpv^Z^X9(0d4XS8!$%mTg_B<&>p&g*1;et zbkXA@SR#6f4}NAlmxu$UU*uP1Lx1=~)3ZOG0AYUqcOC-Awa8gwlu3b}-ep&76U&w2 z{Hh8+DPH#%79{|=&I$7JE%xrcd?vBZWUM;huLCGf<}xG#(3K1O)TwlZKSryN;tY^` z2Yo?s;uk9u99Y(9viq<$P?GxH@B$bvTWT}JT&cG`>}CR`{D1it>QFwqsR5s-dhj zgPX|U2P-dJiU@Rx7S!&QAo!%f){A9AZOSmwG#V|%17V}{@f`4TzCUU?6K`!wa~?QI zd?)##!D3uj&tOGvUh@ap4Syp68lCn6Y{F*LXO;>UJ7^7wIWh>>r*z9LpT1kTLrUdXZDL}x3*Ic|hJO3LY9yBVS zE8lCfl-S@;D(Myhi4(~h6#F&*+SF+5Y{Yjk-+i#j@$}nD<^{_-jux5PigSQU%0Teg zccX}<2Q^90ma`Mb@StN8`8%{#pjiZn^v;j26|=Mr0r!hm=EZKT09uOTQu$ef{fVq56VLSMZv`r5|oZ~`tEbv>p|D-CpB4&Hzv{>6TTdOu{E&GmsS_9Ai zd_?|}4MI%>4{#W~Y0Gu5H{iT7QTZch_@dXBvlhw7jT~9UaRo=(shJ!-TA`5^Vy&-N zwzfz2AJv7$Lj!MQZ3gHN8SK=Zkn%~hmWW!i=A~mDFk=@gy~36Xdf7;a=;uy=&Gwu< z^@AJ;sfDiJ6=F3~(Z0dtffDXH$Iv^55)&5jT60dwZ!`WN?%J3Iw#V5j>4N@K`tYWO z^3pXN8|WS%MH|GIWVtq*JVY(fGO~nwV|aeJh3Jv~SGFk!WE4E4VyGyh&%Xqs3l8Dy z*L^J3S1X3+#^G4#>WHc7u=^(DY^MT2ro#KFPG?t@W<j?G~>HO%u6apGQWPK=iAL2Dt8zv&6}Bq$&Rf|_tVtXZKSv1G3{14LX4ffLyvy8)tU*|eW$A_1Hx z(9yEglwDK!I(n?UQ^S{HXP7W=M^&%cQ=m16-S3+b;d63wrqSf(d7>l;Ze0nfZJ&>!3}H(|6+e8dpeeSz&EE5B=JWf?9^pzH8HYC5kN<=EK4-D^7P+dRj-P7j-SRNg`=v;-G(SpqG|7(kOIeg7X<^bg>oLw6Q z=lN-HRi;=Yt*7KusP)mfTt5PU5-hb5aE)sk$at2i8dukw!SR*8eY<0y zXz!IwBQ(z8IqLnie(-q^}?Nz3|vVntb8{Nonu*~LB8sezb8*(u!^E4m+x zle?7f2o)e0K0>`CO0dDz6#wk9#i*XOF|A5L_()Bo$*>Y)v6hMO!N$*AfwDcW?s#sqGY5M-14vC_^J|C)K?;GyxitYi&xuzoTpoYumOlAphB`q^Z zO<4Q`otU8Y6?sLXOFe8yT6<+%%nyH$;X|4Y25w$%5V!kvgg0Sw`P5G55nAp5X~~w3 zeKjD)=N?O8;W(h!+)%HJ$%Iq)y1MSC4AY`Ez%cDX^CWELj-8D<#Lh2W(Z1t5^#ioc z-pf(le$aKUZkkykgtJ*}itewEzN224URC^o+B?kNzGe5V5HlG+#n$S>_>k~2UR&wCF7pzo;=f&M+1$S-v@MVB=C1>+S9hgGEad#_D~J1NcgzC=Qi3;Sjj~H=dq_v+WR;FA4+*d=4}A>8@m z)S=K4UDQ*C!KXa-k(QcR!rAdj;}XFs0|#SVQi?UR6N`W7CBN`AvI+AIue)-z{1G{t-!*F3)^SM}aD`1xX9Gp21S^ zZI1phNJn>JiyF?Hy##?PHBA4$Je%}$sp14pt%(ve%J+Q8umq)^w{AMGSMR8i--uZw zYFJ1OI^%3jKYXNZ9SmlJtYx2EW!ZST1I_WUf{VowK2!zbgFmeMA~U@S01fto2I|3w z8dewguDL7MMKz-g5O%~ds3h)Yne1Bwu+}a#brq~N{k^~Z#O>5Lk1*hmo<_Z6O!O`r zvg(aJEKCM1a^o;)zF&YA35W0a6Qjkpy?0OCqB}dOX zdbT*}%38wc>BMNMSq7-0vn4H>ALJn1713xqH;K9~H#Gxwf|Uk2@RqvHQmoSW5wajc zki-#!`|=8A%Slj%nJIzf1J%FF?OblfZ4U_nWua;4oxygOC{1Y+8CEGM$oIn!BQ8oh zI9$P-%@2K<6U~`Lden{`4M%sBgpJJ82pQX$(3t-$@swGLxQA-rpNx5szrV?M)JUaX3I^l z6JhZEwj7qBqERo_QhUXj>;u4%1Z_2J>a!1G!8PK`-7=Uzbi+NdfMH^z5No)@COqX- zP0AT}v7HhcGLG0au`zYh-TjM!dM0Ls8pX{g#l}Tm{fb$*@MZw0UH~)}WPcAxqdo$< z3rgeeM)f1pJ3M(uJnf@8!qzBbU`^vsfaw<&=)fAjRow&-E5|-3F|ph?5dy8}Qhi%0 z(>(_*QDz=ZB4L$V&NS*o0-!azZoM^)H^cqWQq zcC1l5pRZRkZ!9JSKId<05lv(H$T>M0tNvL^!O6iWNh19Cymk&>7gb!43PW9=XZ&jw zxmOcY{@9&A{MUlgR1DI|0d#V=Y2-t7)&QMuTVthCTPX|mN+JbI-vb{LV+Dvzy1rKc zKKY$t2k0LPJcQ7`-t%* zUmYg_bcHWSH!iFRaNSQZx~8wU3h1bh_%cA+OH%EId>HbJf!DSDus8(O&Wyu^Ho&!e z2|-l}(4a<>V_d)v=&ee4rOy8Cd(dMbOq*ZGQ87Lu?zp$SrePG-!#O`H*TElPRwX|= zc+&TX!2AeEH(D8ZjA2gZqUY2rXex0RH^V}JsO6&R6Jfnjjcz#>0BHjRU6jIqBY7KS z&^?y%HdCp6L~dL*$!w#e766@N@+sFR2y1ZVezy$;ukqOfd(Q(}v5lrsP({go$pJun z*OlK2Cx1}mjH{%DV~*aH#mBPwl#V5LZ2;RozZxdh%O_yUxR7k=PSvYDy6ncCTJxan zgAfJRgh|kum7>bIx!HTM2DXQBa_b-twWZ%__ek<4o9cjbxdA_cu^j6Y6H1QJGTH z4utJ&)N9cAJ{{Q1YSzhgN6gZH&wuj$DHoRgftt4sfbx-vjfORE&cA<@vVZ&e-|Ef} z4KZ_qb3&Ik=1>T5zs9BZcZLo5?Oc^%Ts2;nnbL3xxxj5WR$Hl90 z(RH6QoW^~Y_4Ov_4RWlMV7jKoWY{k2#>6kk-o}d+hoQQ} zyxvBOnaK;!a$_7CaR!qY9&R@4tpo4kqQ}`r(Li`llN!H8H4(agJwVuv>NLgaJ+k8V z@L`vH&*spvnakVP_##iTaT}hQ{0`M)tfOk#^?$tG_fwPM-!J&|Mhg%ip&5F|(7UG4 zo1r5jh8_f!CdHZtgla&VG(#^c0-|CM9l-*Kir9i6A~tN;(42hFKKtyMowGCh%kCeL z$>dJ%YjR!p=lyzj4PwT0wPTo%u|&;#!c!Q-OGdHbiicH&z;jm@mG^cR#?2#WZ4o=&EiDndLsmn%t|o0uGNx~^^8gbH(dY_m1H(*J0y)H=M72dcZfvj= z^D$fuWw{TYKOc#S0^swRi2Z`=7d2GM4_oB`KNK-e+IfgEYn3LT#PxHyd+8CWmfp^I zuLuDKBw@cz(@un!mYE)PXFRRE+VX_IA*+A8Gl+!nbP3$GvgNF#{{#nMbMeWJPM2{g1?ch7&?VchU<*oxxNPG2u7&~jId=KCr)doW6$<+LZF$JdM z9CZ0o0|&vncwDJ_HM>zR8K}}gGUn@ncycP0z@JHgSqqIz;^z!K&h}HayQzT)LG|-)2xCRY_H2h48mLe5IfBo9#(HVf~ z(mSJbc2e$opl=K%?%+l9!D|EoM0?@lS(3aQ2`7Hsi&;XV4&XX~*xvvhYnm)~akg`D z_B0baCe-mxhZJ&v?+X-P@jZ`qnH^quDN++wU82*8fXdd#zRq{%O|?U5C=M-d4*HnG zkeI*M=98Y#AB?KqcWHEe@z2EKd~=d>%*%`1fC@G=ly>~u$}yc0N`~;U_vC?U2Ew@o zEOaJQUJ(t?7GKwOmVdjYC3dpA8*F|JlEFX_VwO^D<jdLxo$60i`26dt>)oCn8>d zD^05D^{k6XE*ox2$_3FG$UbN9V{2qrfoQDD&ZK4U8rn;98Pfk~{|LHNq-j(-2aU0( zA>v+8x^wD78A!Sy&i5Y>*8p94y=7i{x34;!E8J6~Lk<>v&TD|u`H*=J^1soyTXNl< zY0!BOd|+VBi(dDSnEd)`7vhk$W1!}?<7eB#5p@k8*Oa_MiHHpRRVPTxw?7n4%TVA) z)SsLWT*0D8Q(%N}`C#i_-?a}8cuymKQ)tUwECZH&PTKAvNXCRbqTa^Oao_lRVy-u- z$5!$$oxX>L>e}x0tO(IVl)N3$OXT+=ezF)A;QfHpN?_Kb5rC&T67Xv9q^dqDshYtA z!xgsqJFfHCY$gm#aHcj6ju#=cF_!r0CJz=2gRBM+`)-R|(yKWZQ zU0AtW=WzI+q<#PDOpDj9kF@O1w@|!2WLLE(RT^2) zT;jJZ=YidPAg*2>`_@KQai*4fnf_gB_swj|2O=GTal2iR2Ft6EtjX**!AX1b1|cep zR%R7SmN?U#=(*Mr9pV4vByH&71VFhD-^Yk5g0}5Ccj-kzV)xaLlZof~cROlN;C^4N zJv4?&#Py+kVQv;3?-(awBPOaNuV_KcIghp@x1n5znxKClFaiRuG2 ze$P~HDbyO7mMRqC8IaQxUswRsxGKB{{n4dRnEbl=9eFf|r{Eo;xkm4z`&;V47X9$? z8Rr^3`n}|boF6s4xLDrbocFOv_oj93TDsTvfa&zQO&zHOm;T#t=W~n_Z64K~?7F&j zYHRuz52?O;C~R-)eV@$d883@7UmW`66b6r)w%?OD4xeN5#rSl1-bHscYqO>5Xk?Gug?ipKp zGS8~UysyC_ZS%XY&-}l@F;Eq!h&%i<)uwxnq`Henod~3vyW&$~~u~I4*%N6)5^!WrxPy zBw8vkl*i7uF*EtYgFDtQ(MLavM|=~LTC}!Yp_GtP%gWEjXys&{A)8Z6s=YPWd7*sg z;I3%Ko#hCI|AT!q>i7U+vP8I&VArn_F)d9OY(BuirtU$f(TTnAx z&-5mYTD0NuC%B`*>CLJiI^59zIWOKwZTunM>9OPL0ONl_qPK~W=)nI+Bw9voEfXzRDj|3RW(mj?zXysik1>l7o=8E;=#(Q*w|s$(h>R!-5| zI#+6vF1=l;O&&9NQrq6a`J@6qzqNgb;qsw5%y4*HJn#iUbr?VHC{+A;dP(z`1; zn|@$-t-lKQj>!6;AGncj^!WRW0x#~=OUybLv~$OY?~iGDCL8lHRr@xc(A&>#EF@j_ zF@Yu@a0-C#zP8UMFlEX_{&}WIG@eOGJ{fe`g>4g*MYDx&3lJ2X&&&#$c4~MtaC|&4 zo9u1%G3do1d2a2BsUPI`_GY(7cJZeq<_6y$3%+rPlD;Qm`d#36Z@(4E`9RWAYvRb& zgWj=t`~3>(>uGL-Do&qA`p@`YC1TQfS)cT7+zb4u5c|);b?p4j8?G--j0irGW{z$W z&t&qh{BAjYUvA@*g~GqXcS8;Y{GPTMD+M(j{N$DggmWtRJtw_6M}AXhM1w0*ALWS z2isQkg(6shZGfJUSc>?FER!CHWNcj;(ywOh$g5)@> zP<^3n72WWU?~hqE$Y;A;<7Lz4(+-4ZqJ9Y72vurW#^yD_X?K$au*%;b8^I~1}Bi)g6Gl^?IHD`f}qUr8_ zk4?C=a^zz{y;=EC`R$QhzJFs52T{e?WuF-kpgfRr{;*i_m7K}F{Gam=ky0`-fehKPLsODj=7-ofMo1E?&N;Rhh9;eu z0U;K>a)^C3n-@M52Dt%Ec_8@yqiR$!YiX>{2No~wfq4p8YVnK=xBVaX2EWPmRsGx| zvQHIq9IyV?IRyeD`asWws87%j6AO0|Z1^z54` z{Fad+QL4e)K(Aqo4HoymsUHace+NB%9F9ZsU0sd=AYt##UYqFk5e1 zhanCZ8}zW^B#O^1NTe%4i)-hALFm96uh%vR12k84SbiL- zK6fS0>6&DhZAzWN(5-Kax%V}0Js*y#BaJO+!_I#-keRVbUb>WV(8Et;J+IJ6N#oITL%TIr*w5$aSzm~Lql9dg49|q5VYQ0sS?HX>RY)|sn zm(dy5K=aOTf4Ok<81uDz6Xb1y7vp*15 zcGDt#2y=22^dXq~faw&~+5(Q1L^lR`wAjSr0O0U5Q&6a6y?pa3Bwj>9`SZ-c4E$*D z)B#C!LOh}_N)j9^XDJKR1tDSc*gr-TZl-m@%s(pYIe}eJyG{@ ze4uK7Vu9;`;3x{sIk^Z6+{W3+z%a$QrIW^SX#S@je%ipa25z2f&K)tP^N& zt!BWvU|4xFDi$Bvy^CHke*8{CGa!qP z@+JbLgohanc%&_~TaY5N1nc3Y6lcP2e)qD)5VLc^Wki&CN?GtYRsC_60~-9z8O2@S zC&Ghb(Z^c&4DcKfp?EAA@Aydc0GX0}6daA%b}`skz`PFMHJS#inkI`My4Ml1X3~OL z0CsWDu3y%n@{!Rm+)!e3IF}#3Zvr~;O$MGLkDAVMW`K>`cZcMFm#hfwt0}hPaG9QE z8I_J&2d1whx~CD&f{2viRNG|4ZY#F?Qs()_-I|XA$`@&brE z&JU+<%Y%yPT7S_-S#vTs6zvs@rqu<(9tFhW&|t$+X1oBpmF^|e4A^dms)%)Si#v(3 zLfKLSxHbj00{Fsr85SBG90{mQv6RLQJkkH^v^ z=aG|2@D~??kLefLG7zHi?K#{-RnxrcukNi3KUH_u0ta!jip#k+ z!YF7cFd>Afi*!;1r%xc&cxE^OQtY;dqQT*S6tjNx=8B&!0D*mTScd?;HHQc#K>bAg zLqqb=$C=2G9*Ez%jJ6xF9^Syr^%j)_V|m%mXmC8kGc6AsWq|M#K%5Eet&$1CSH%6_i`EV8M}yB zbncaLWEL9gFwM^!4Nf7VLeb>X5nmAl-XMe-6Ck&hpqtGAR2a-A8T`mSRU>F`76Yt9 zgcK(um={0`Z($Q2JIIqNA~~B|-zv)R$SeY+xCh~W(jWIG!AO8Crpvo)RU-9v6a>MV zg{L*Ig3q4>+7KXOt2r;(RqaAcT0tmhB|$_4r1McmyvoE*REFrgud;`IqDMq>n7#Cm zx)6LrrJMc8D6AZf2qD6ZF4kx&f{zXGOC}%z)PQi}F-bx8>3M`VuYTZd!bcp`j{$EX zLUy?Um#oNU3}~J^{Xg6(8frshKg!sl#s)usjCf%OH956AkJdV5hce9Y51^rP2H^af zFbor1Cp>K|UPtgqjfVh(*3!vV)P6vWQE~!387x;>gaPE&@(2(!#u+td0A-rP`7L8Z+kF$Z54IS{i`}=) z88YxIS|bPQLoYX*24wXhaA-sf)?X_vtU<(p8sj628KLeckL1x>`YXLLvfw?+P$L1O zMNs0>f{M~XkmkK3S8K}vun}3F$-BdOM3$Hxy<>9>7tp;K04l8Uk($odI@Mk*MB=J) znz-C$tD*Qx!nSbV$a*43L72*yajHgLMS327C*Sl9)xbu)5eJ zng%!ovyc3b0V!cB(kud# z?LTz8L`ncE#pXJ!Asu&to>p|od=*RO=TL(HYmZVEDu7c71irrVW-<*faY4Pk^3#YL z4tzEQojY=nnwASc(i5!V5+2X!n^wH!w3gKJ)hiS)FcWkSS)48IxgkY_H%|v%4)Cmh zNBRssd=r-v{iq8^+c)(#F@T6-;V;QtrMpL$H*G{;UqIx0$}T-b*sONYXKyj z*_Nt*`Td4R?7P4S)jbm5V-5#{go*~BUrf;fI0{zTTC3!+`*~s00G0MZtr-M6N=&aU;mR#KI9J^rMp>AEL(L*0q?Z% zj(xOir>`N4VE%bM7q)RM#DH-NuVXeC)YMrTioOLaf=3f>W&jY8KSS(CK$-l~$k2c& z(P95G2Fjaek;Onco~hs5lqqDOx}Ta6p~o`u9b+y)1|#8zcDsc)U|D%c?=`Fg4WCTf z2lE2#EVSIUf^-n)VRRxbR3mNvq`a@m=o=e5GSX~PJF~sg-=gkv2Vrai5Q3Yhc;c=e<}dYU`%mlBX_ z-^Fq1@J5^fl49o`nAF3O0LH?S-jVYD8D5h58v3MAQ0hmnSicV5!z0ZDf95wTH! zx`Q0;H!~69WU5pTg24zyVRQX&*6GEfeD5f9{7Qwy2p1KC|0T%MYeHG&6YhHG|N4m>X8E61crN-doZVDsNH|TMc zk>&G{qf#{!?3Guc)34m-0MXNTa2F7Df(yQHQBNg2znfGBpqJf=s5ru4bMoua{Q+^8 z!1aZeSZ*{iH`r*h&RFlc#A4c7TengO9f#&|Hr~B`Xd(HzyG{TZb3s-SF8jW}ZbS9V zBEoHm@H|3{`{n1*MsLQ`VH`XAVy)#PZm6pV+?jm_e}i0XbBy!?ASqBWCZf_Ab0bzT z;+VKm#1bMNkt?w@esMDH(UbR;Xa7vr$!xT`i*|z+TerRbqiU?@mTB-R-dWw;sVvp4{SbY= znQE(>ZjM{{_^)_njU@l|qQg|Y%WY%>0A2M@s)aM@14oHERIgfgHMUD)%q_XI>n}Bo zb(Fo08$B!9-9)A-ng@C0Jf=Z_=dv>#4BN3!`7GY5jp7eyvj+3Qbp za{|k|ybq!R_x;FiH+O-1mWeFQCiRVT4a6hhYkD-HDC=-kid@%{bUpv$ici;1qH}%d zyj8d*7{8-P_8Mul8q`kd%Pv`{WdMD&a|Wa}}U%?tt&c=Lwnk4KQSVk)<|C~BW)K6xg zoVP8c-n5p!5?NYm@7=NW3?O>2*vnY6Rd!@QR>vCRrp?fNR`c5p?Z0hviey?gXE3fG zh|8BZwj)6cVPBe>J8D~^-ftmL^H~PaU~GYbv8&y>#k&=){4sSET#ginI%%dO1Dz!5 zTPy1?%9+0D>vI!%{KFZt)h@M6cxx{$c{g9f2&FG@=!rz2)Aywha-L3|go`k(NnzPl z)x%UGM_oGy;~}>9&7P9KYHtrgOyN-?^mX5g{1r_XJX6_hR0_5MGGn_!vi*R@j&{k- zvU)>IjHy0NFnFtn0F{yVDQ~er(q#p}U~`k7qsA)_VJHD2dlfR+ zTpITTui(t_#Ot(k&PFjFgM*--w{Ws7bYN%DF?*p7Q?_RysGwo#0J@I`i@8rNkcL8? zAJEh>j-m?8P!Q(CaC@k=(S@oIV=a_1Z0(n63GhiEL_W>gz06erG1h?gq|KfO>{v6? z6qT!|q@hd?!N7&j9gVwWeG4KwB~MrHrUGvgN2*SYZaYaYch#(O?`;*A;@&W5oD`Um z@+?#MT>Xw&sRq%a9m@N&5S@qhq%X_KIK2Z%kd!rE^JbK>P1>fYE!?R6;?1;8zauG% zrBBV@y4BRXhqGemp%?ojr9YB)Sa#voy|8Ae8?Q0L(Ofyge&@{!+k}zsmsA&ecB}cH zylve&hjtpO@(p0d3kVg{rBkqJ-2*bs43=YanUdsN57ci7s6CVW7xCsu0QjT`Bt4kV z6%rL_bWhOMUfGBWjpTffh$weIRu%G|5ArS7=byMNg6a(vc24dFG@?GWyxkR)Qoug; zfEj0nH+}u({uD1AukI9YCv6^tbB;A!?qKOhG|LPnX(>w^Fp`YiD~AJ{0E!HsXfxW7 z!U6wQ4d)o$F79yKBX%cqIi86hqWld1q2$%sBpFUny+0r{kNEpW7fY3_4j-}1T|B?v zD%x}HS#pm4CRl>gLTXsAD3k6XDum1VW6TAR4TLYRK}$Q%ySeoHvr(z~1i_qp^U3k& zdE1=rJe=Qa);*OZxp+Q#+5oVqxx$}HEJmDoj1jMm?kscXCIQw_q`}oM$Agc?c!LMP zl{c&e^2EIYY1f(DhQDlyPzk;nuo?4Eqsw>* z#XBSAH>4IZQN)v=g&ssrwVVthO>d;S2X6D8hENzBl%@;r3;7(_p|^9ymQ;4swzFH+ zXbcVOl&ZguV3wc|rBp2i`-w!Q+!J+5j5)P1R|I{?DHhh4dy7_NJsn7k=nhi6!sulX$JPgq3Lj zS%$59Vn8VW?ayy<6*sy$Kxh;h+;kBHDr)qoQE`cd_kn_hq>LkAw&AP!Am0=+S|)vW zfLOe0u@c&BZ4Ubcn`OFj_G!u|jBGDS&-LTaZ5{l2fA?7xxc?E$i+BZJx^c^OPC9?v zYtrdOZQD|{EHb^XQ8_0<(M)kp?*I)fUqe@#etyTi>oQ3y-~yYE4foqA6FoSnS%rp) z@AdkD5VE+r$}v5*=jvaH7z^Sl+gOIFy#YE>&b&(US|!Hz238M$S>;!DhLe{cQB!sf za1_?6JsFES-ung=K>ZrYm|D{Tl{$U9BkFT9F&##kHVzeZja2E(57X?68l>qdSFd^c z2jvn2evh>K;JC>yrzVk_kW0k{F>6up$+Du=7MthJg~vwYoR3!Oxn2L9-(X({?30C& z>WF9aP5oAzo~3Bb?_@epf3=}EzCyV`Jv~$`Lo~hboro1W!|esoK4l#zSmjrw1=X&t zNWSgQJ7-nNXxqs{k7mHKvA8dX+caK6k%Vwu*p< zq#(oLDSU3uPfP{0ZWVp^_jB&yJwdicO$7%h`92`YMY$~OgH9E*`fEbQGU8~zr>!gI zj5G&fa|5D!h_2LAJZ5G?8}i@gm123Y`qjJFHY0|+D>RO+Z0WCMmFWlemjtznYg(pH z2ezO-AG`=wtt!SI;G0;aYchuNrIilZLT}jRN^};z(C@S2!5BabnP|Whadvq^R=CGr zKoixED}X%T3AVHxLc32}@p048BO< z=htXf_%r@`S;)W5f5vC10KmnxZs*988X5qQfI;u%)$+kRs)t|#K;EAdl7uQV0YmFH z!(gJHUD3G}jbj$#JJyC({M>fTRbgMyW5Lsxo5sA-~`91rTX(gW4Z}QpIICQGPBLe#<6Ve3rey$UezV4c3=~GF#tM7 ztma~?SsK->x6$D0x(bfK47wAw_p6?-221R?np2@9F}%tc?MOd`Sj48A$=;i;hF^i0 z&Yvjmk@p-lL?!pRPPA$8ifp48KNxIjvE#P9C?1^u zr}~5-AG!wnX;T{rQ0q~z$AYv34)?M=3!g~^8W_qnK|W3T z3=gu7mV4g|P|+knan~m@e&?!hLe2@0bVJt(IWADWw^X2&V@qSI=(AmH zbJPZqB|tA@-o0o}d+qO9$pPJbI>4c`UBP|qkfo$XDMu}fDn3FDv+OLJX6ggL22D*c ztgGh$Xtn?3#;T)&M$Z1)`~KOuXpQmfn|VeZc8&zDioo3?88=IZbkFpcEJMPdN$~8g zg3v7KY7mLR_Cs6hft7~oHD0{>ju^HajVW@Vb1ACkX}u1|G(n5Xtc!^d&uQ+ji4)jR zphYMogw`8SS}Cgl@dZ?I++B(DOgS`rElSc-IbXyDZ5x>CdMX(pZpq+r?8|D{*MXyU z=#rUpG65QFKW!Uqf!O6e#Omw9&jC|zoD*lTh_~`^Qij-qk{0%aw&ah!cO(ax0VagB zQdumYt-Tw99mqGq$`bU!Btgkbke>&UrQq)Fc`gVw4<5*8Deyrpz(N@xywX#dTYGsQ zf5H9AI2wT95MdWY1EY#{rM3X-*%AtN9JvleB z_!5zQ1(IC3E`8mnhqtBwlIVJS-)j{zNx#$8QLvgM(K6(Phz9C57P$_v^0BOO95j#$ z-SU^R+nbx)bIp9|vGn@vrFqD%laA%MoB;RyYx!Wqn3@K_t*}^tCY|d=%{e}CV%!;V zsL$u|q((b;(06qw?wc*j-H5W@Vxi2EnLI&3@O`cTo$=5=+N>*BIdPlC)UU^0wrt$K zUH+jGj`K`a**nLc4(d5pNdLOh6v~`PM;M2e%DrVwXd?$)a&qgZpX)ON)SyZip7kW} z>UGLF^aL!z?lm@)lkHUK)+cghHj9}ZjMKu=PM4V~k8>&Dq8GiIK@%s~>xB+Ikk)XG z&su@Ma3E~fE}LZMood_@Xzu5IjOnbP2VB-Zjhky~h(dhmZ?2U0@)Q@(o*ZOjSCO@n zx8!MZ0ca$#6<;qbo+srWpAKI3c@eWyW{b|)MRJG(uF^={7uU;j35AfbEbAvB54g82 zZ(pxo<52J_eSEf=!H8c+gbnScZ9a3TVoR9MisPWsxigY#cx?%j3Tg`>uMu;33+TT%Y?Uk&b@cbfz05S?&i7z(Z~IOWT_y6>-gct22?Z9GQN{h>`iKC2D)luB#nAl9Ktg z6Z5syu^U&Up0@JdU|GIYjc5n?K?L6SEpo>I$4zjnBe60@*6G$jsU$(_dDs$+gt(F< z;mc(O+!#5|2ZyNV6kIR*mr<ChO2RgzVpi00vZ<)|X~3`K zrqT++2VQ8xCvK5MDsCr^%x4U;|F#CjNe zJK>o1V1*D<;94{{5+!5|G_0JI7xp?O>_shet>*R$b~V;g?bV%1Hm+K_t@24uooSsG7SWqj0={H;&kVsS54HbH&XW2 z2Nhds)4!m6kCjQNVtIlmh|wocB$w+uUL;ocO7!ZAvN$D~_>pD4byE|Gy=m!4-{Ae<|v-e#!uxlWPPn=e*< zra%3+?~Luczck>HLM-x^{LO(l;({*q$q9iqNhU!xfTS-hW;%UPdtX}E{lYltS93mJ zA{$l*Hfejj-!vb;kDG$kqa+OM=T+HJpLodRuODbXb83%Q@y2Hzl{UbU!oFh}@zx)^ z4!Ndawx=i_{~X(0eT($@8@5j&@3WB0j=OH53Rk@ebnJXouYUfs|HqUh*S+Y-t-pGz z3t_1sm($JVeW>v`rSZ?h#B-YCDMG1|`i#{5pVyXXWPP~fxGh`6<|4`mZ|wnnRe$rs zrln$>@gZsrU=0}U5pwK*tEeA;U-lZKw6#r;jnYZ^x~_70tbli)+_PGeS+L5~B0Ivg_VEyKxX~^g3Eq$hL8Cx9Y6g=`9$h@Rz^gXl*Xrqg!a#o=Q9jbld6#e z^~uMAHUaAjfovcAMQE&2wrqF@u*hbk@-D8p15s{A2`xo^{=>)aPLU5gZKQd_FcqQ`lfZe_*^U$iI74MX~p-hR=!Q*9nUdp6;# z?!7wu%vVJdU&X&QvWoq;orK7HT$e5PHaq-Y&*u2HZ&TZAr9XQnOlJRwweLprJyGLj zCW#yG?{%Qg6ojvBbOe;|e`B8G{TG+!`Tc+GGfJ+SD4qYa>rZOeBzwE|%cE~o zmS2tEiv4%k^2hrxZyuZ|=|A{)lTUedVrPiR^Y5v^IS|xTAlNQ>*5;Xu8e$|c4b_=b zMaLJrM!Ga+yEY`tW&C`WDVd|cu*b`cFAUJqtlmiiT>Yt8q;YWl1Y7e`=uDA+MfuXv z-HTQ=!J?Fd&8e!dxl6}E_X3yvDCQ%}M?F2OfTX}~mt_y{ZR4t5X*f|RpDq zQ$yhxBJOmqg3rM?HU;D{UFvK$5*eQj&6+8&B!AlhV=5}HR-h_8;=s7-0cB{6=L91~ zYs}W5)y6I>p*X-JeQ0|fi*8Wt==t^yO^bA8fvolociK1Y%N$Sw)dMJ{A$n0Se1cy= zPM_qr+QlEQA}`Y-G5YSy9kH)}qRF9^p^wjNFC3c9cOm_5he%xSVQ`}382T_Q*G{3A z+u%N!%HI8xS}c9HLvR^Zk8F2aGXG7>9&Y^M2WQGHw4CwjniBK{MXk#;8MJYMaFq@Q z@T!7p^&ko>9+w$IeBB9bc|5Be*{V@kw^$;H0D5UzE{MV^y3`ysr%bLO`h?!7JE1)n zan@WuSncWnQM2djRpSj8>Z&qFYs~0C$;N zKXqQw-=o^4aUtiOLfgcbOL(Mx|Mw{HZjsO zODT?;BUdoq0uMu7VvoBa$(zvu!unmG(2NOwy7KHms&^FB!n$-48(VoV7!1KPCl%fH zEN@K=atTTV2WPQ3&G=y*%>s+yX(Eze_*5Utit~xab+*VAYRNm_$&~C#mIJ~cxG_jD z3}T(Ng^mp^U)MUbMXT+UD>pFPFEOnU^ff|cd4ST@u+BBSoF47DC(QqB4q*_@S{2Mp z!pfME5T+UtqK+xoJuAf{=#htYx?MH58}Q|q=$sFM!)F~#?k33l0l=>q2)HIfmWvP1 z@z!c7K`ig^U0c9xJ}OS_5r&{@$Xd#ux%jJSnY~x{^iGdn)b15XHH$|ICwXn_J3F)@ zqVqum0EwO9Y&9c>7mPlQFin@%2#g}3oB0f@S3+%zbV5S8AOuIBVg48zl3npQg!f`GGH(j*I7!6@(1;09M_9ti2}$CRiGxF1SK@?NPc%%KO@ezPO`Pz1;r;@ zt=jMeY6*Dn>0FJFvHCf?4$Ap=J1KH4sC{%cV$5IY>fxye6$+H97#JM|u4%u)2xdVr zBx~`7f4m0(PaK|*^tw72Y3HVeh+%FmVzPhd3os%hcgik#!iPBz8a#W#WYP=-O(_Bu z)5dMb{6r-w>CGy)5M4BXrT!cOo(UNQ8jm#=$Hg?O9&aWhEn9i@$a851Z`X=wjm5Y~ zXJ8i1O&r#fHI&l8Jz3?L#E(cv(a|o~?EK*SWo1Sh&v*Nk$$?$Ux2l!KZAY6watslk()`gKrgfm_IB7l-B~_BNh7K7NE+Zc>r=qXQF0zTCJjrwY%&}4fkgj;XOOwV}7ul^= zo7kTmwFDF$DKIz~dkALgC}hS0C}36UT&NpfB8(tBg5l`%zNdA{FC*a2zVc-b0*<_@ zQHO#p>1-QcYUiAcme}NzpH0Ucl8sdDa@Rbw5HcJ8<(sc1HZMx56pxAM+F4ho1s^YZt{t(`m+5-=`Ipgycx>E;M9Nv6G`RfzahJ(ucp) z7v!ZpBjeqJrJRb21ByssRKr3Y&cZ_d%bR751qg6BkrS^)7)*m2N8eb%E0+sIo)S(a@hbCA8OCO9?2PY3#csKWd?wZdhHWi}RiYr8&S313(_yd4XT`>} zCHu~9XXGZxVA>tV5CeM@hoa%F9kTfchpQqvTjzJNlLxo1R#*>^#d~fzf^7MoYCQmw zc7Noy{P4;s(2-VoE=Fuslk9$yt+97CVuo*K<9lt9YNd_hx7%t>o{9;CSCiFT*@Rpf zc>&qFrNXQZ*g|C(h{0(7L(GKOz@#8h!CKE>X$_F=Vw%-IoYHR^{1JJY!1vni!!)zi z(0)Z6NglKv5wXL7k*4|#M4iVZAE@XIb`XC>b8-?zV%ZYxu#DH)Jz__uS-i)A7Kzs3 zAWb^WT0lO#E(J!AEs7}C;cUYZ)C)WaQpq;LgB&(h!Q5HU5s*0{#InoZ7{G?}iTcFd zN9;oRXok!{1cc|IvRS1|1zQ`lwK@bVmP85TA#<_Gx=vP!d45?HWaJ07a%9`Ike}cL zs#J=OSbdUg)k+c=AM}M=5<#K6l`+JH(IE>e+bUE{u~H1VUIS=sKoodu3+aeU7#Ir( zBLm>oMBVivRl;l4JA;JHkS8fbbPFh8+c2Sr1QMvjV?bt(V8S<`spRleoCsig^ffV- zC6f%;^AG$nL9jBf4*8F1DFBBygl+gk5dy?C%Q%!cU61+tAufG05B`gcdmm(42W)Re zDhXfV&_K-?u^g%?ZAr0s4@B>4G32AcS<@!TK=~w4`)Fg=^)1!CqeXQBeLOq6LC+iu z*5b9cU?~=x!&-Fk4-H;9(nnkgwAB*4SOicazBCc@tAfGXPNxWg!|Lw=s)*s!e+!H@ z!Ti5i=(Iq6fN6LH-7>AJ8p9@(4=NOqt%`X;W`5rd)B8o*3OQQ%RtIsubT+}C!BqJ6B zb_Qz+0b?GzHo;<=cdMAYTn9q+ z87cyBg+k%2&0!QFUVL5D>2zE#peA-?YYf=c{glQcW8d@WhM_@Q#PD`!ph4EDfEePH zd{5NHrpQJvXq92>2oe`)fsFaa>$TK}`RqD?y4cvn>TeMOw+NQ<#XhSe-(dbNCS6)~ zxgGCJ!l8$)1f&|Vc)STxN~aiH1=bYP6s-JJxfIKZAx!w-);jIM7OD$Os1-h><){KB z##jh|(%7MhzbGl;VBSZj7)b`<8Vdn+B%2A)z&6kpZ*->kzi;xOiI|O6-K#opn5Ief z>j5}Ggg9^J=~F|>W&q8=H5`Ho9|X0mhdXQz;ix3NI=pAWtc2yT9!Cyr}96zmZh7-TRQHZ0~BRg;*E z)$-OS%qc*@hso{rCgGzv5ixwaxi5UCuv1|*_^qKLyeMDtriZN$U zcHLwvR`hxx`FXaqgCv;gs$(>rmh~NJgRuLmWP6MPS7#ouSnN14q z!(8xRuzX7gh(NL_0-a7C+)Vc!c^xlv7g4gGB`0=v=)-~)1Bvuu6Ra~lP@f-LT&aOp zKf>Eh1xOHL}phI?d%3pK59<*YC@M`mbdT2=>9 z#Em0U)54kBxXG-v32J5=ZK+u|YQxHk?#6B179ZX}yg#4s_fO!Q>u}EXx}LAci{1!z z@^+VSz0#*m%RyPNmrF|c`{prQF|5pc4utd=iTG zc8@EFcP}E%Kn>COll3s@3Ke_jo$%A!XKNvDUo&sW3#pGUf*#pz&I>;ioMO9~nQy&= zRM`EV;jUkxg5JrAFv|gN$6!_s1{bOt3=od~yW88H0@Ljd{>qa0#+;K8y(+Uxt?GQp zy|d(S(4NR^S|G5|NPN%Gx>wWa0dJ=|uPu9l(%uDh z08XDWYfDi3R0(AfCmu z0J?69BZU5Fkubr=Q7M~dU%v^zFV~ z_G^gGriY2N<;bVbrBIIKZGFgsi6<@SoydK9UlHf-Idnl3&W^K1^o7H@xV>o($yde zVOhtUp6Ui_wTKx-z162h(G@h%J{!hlbN|tmma|CFKZ4XrkQ%1czYegyh&~}Uv1Bx# zqk(F5FeCbst<$gyxz>4hrzIT7c6d5UV-y|NcG`yXz#zzTh%uS&owEk z5#vg(TO9Nr#>eHGLM@z^&%|9GTkk~i;TZSYFg~xddD`UH-o)u3xglyfSe~)~AeY@= z)QdFR*oT2jDZ|c`G8;n?no88J^RX=d_j3JXV~HjAYPALxxO8>nfd1=$*zn=RY)#RQ zh3V6#oe_*c4?;eeJhXfBA)| zkm*Hl%Q%{A*M;jhm==~=?G6k(MJ!_Ah)*bBH?i~)DH*>iadS#RaJXJfALroZU1y%V zhk!)tk9ux-Lu%j+ialnp#F;o;9(U4ZQJPidbHZb6m8wc00HOIgmJ&qGZ||gGE!gCz z)8Sy+=PAc8c&d2VN@o-5UQT83mz~3RX!2p_IM_|gzN~Hf29B$wV^Btxx1-sxTKOf2 z66z_U?s;0Rr%&3prU|vR@B|6c0zcVwPNMF5ZAFfcgHun_TTkL6^dHfYmpgfCr%LS0 zhcAVIt7bm=$t7-Py#B3lh`#svE!8QOXsXlpn4Sn7Pd&9I<&k@9O^_PC)PF~if>xSl@~qikFD^rz2Y2qs+8dC+m>)Gn}H~<_n1%_z`q*MDlMd|f|qi|Djy^)RqS3sk1s%m-2LMP z%FG`1B|okUiT%!o731Cis5WE9KE-ADCiaJUW2G0A{BU!sTm}NDkl}o4iwxNHAZep} z@R(2odN#=g+pVxVdbSiz9DOjQQ2)qxdN|?P;PbJp*!OW|uEg&n7e&x_8foJX`Ix7r zd3xqb$Wd7QHgO273r7(rJx=oqV1Nu5L6Vj#5J^k|s?G9hmucX72R|)8t&=jnysX!E z8(sJP(cho_?om(V|02;ot`TQpu&62TU!};t=c}p>Xo-$LKHof(X+vhQ72EY~YU1wv zOfwrJt~K;3EE9UT5sR}te%=~*k$Xyhg7AP{IE2-K6yab7`X$D0lY?DNDNVQe29Y#& zihlI_QzBS=Ur9FLav@T9hoFo{^*)0+PsnPpUm0%bh}xN>VPk7ZUIb3#Hi0=3WRIWQ&a6cP2V-ftnH)Y z6%a&zV^sk*YGG6xk$rv>#OscZIf!1tUI@j)Q=QC7xnxf{0THlKUA z?IVENn6)_Si;UuQ>PN}w(6VhcB|~mnhe}4rH0Qx$O@y!8xlnA4pBWHXBZY<0=68u* zdzQ=p308*nOO0cYI4uzIm9ykuc=+CZBmv7DKL?AZ{XyJ`u2_kw%+#&E$lTVe(+b_y zP8ww(@f*cDzKXH0uLY~lP;b@5tSRAsYr?+^H7Ok5LAK$@O^$7wSy5t~(8T>JAw;p? z)|aI@bu|gQXm7Oxzm=1#9VtUGpyJQ$ywN&}A_cOpX*PwcyD3ZN+M`D|J`#Uif#o&q z7-DLob(fNPEytJA8n#J*V^IW(|0OZ(Niy(Gcy#Bj) zHvOR!?1mUafe2DZ7ckE1V;fp6uL7)A(5e32vAz9n6q^d@8_65Y&HBJ0`!Tcjdk)Hb z*j5W1nYv_&iUWVXg*adBNfP=_1hlI=4{`3RD6~n?MLpxcl+Ca0-U9=P9>!;-^Ovs} zDVM+heM0%?xAMm74xlRcY^K$=<(@Soxkk;SU(^bvZF+UsD#A3R>=z4S*jy@ResP#ol_=*l@cgA?d#_~B*3*FC2)zqFCtq4sl9Yu4# zsyhkVHT!fQbtaDE3rb#TzW|R)D!*6j82UppiTLlM)i*j&C6Vib|^3?KXex&GdAVeXpiIp+<~XcvpyE8M0C3tvH};CujEH z4|~dkKh-^iocBia5?EPUxc)XV4`JTgSZP-T+qJxXGcE)qO?)L{1t26-6}^vfs?-=^ ztx!^rUb$mj*5FIieKi|#(4>IJJmLij5g_e-z2OPvr$SM80K$U^vmHs*Tl^of3YO${ zZXuEt_s2kJLsCuEw=(D=8VD3;U8?JIp+ zGS=UZJ|{p}Sz{V|LmW)LW%;4DLu^;e@Ck9H5UAUT8&L^qMNk>L(mx!*8D&4fESx1G zCK)e1sG3dR@RoW}H`i5)xbB|wMuuVt_BdFv-sw{&Es7Z#)QnLr@YC2u?|8`1De z?X^@)3=4Yn@fx{Bid2H!ehxSF8i49bf*|>!YHa;@Sccc_&uCN6^d(8|SnN&5Ecp>U zEl0&qg$b}ai!3~ncAJFp0hs<4f?=dQyizHR7K!MT~pExD4 zar1`zJlYqSxrq>eqyT4D^hz_}fvh5^<2~txo39l~N{B{-eyVAYCq3vz%;hmx@<5Lj z<=b-|YfFm|2!&Iv*{fMTtUb{yxpMhRi4q{T>JCmG*?)_K*BaBrO}?MwJ}w%qL5Pr$ z(Kf3-X^~sR`ZU}f6F-aS9;RO~#3q1W<2}dUG1C^gLM2eF&(TjP|2IJUfuBn@5frSw zyAvws;rBWgxHUt(`+h|}q6b4F5_w2}Yn}@(p1xL`K(_aU7-An=?UI`4h6#iL9`==j z@fttV)WTaAukzBDOe#IxUYC}hFJRtmT}!46)mkT&zji1?AgqK}N;f~jQCjtx7I@dLnRkkG=*YY87M|KL5F)|f$wc!JAao^IWG{bWO&^;hW&D@m zP=B$l=3Mf2ruc*u#}{PODXS5sm3$joXX&AWFg@d+H#dDqUYFY1%GDZw>Xwzs-sL>6 zO&W~$WwJ{%aV1_hDR9!(UsbvjmRtW`1YO>cyTcIx-X-px`P^(75G4wLq>N7#sKBaM zx6EwU(ABn^wiOaa))#| zSNg5BU#aWn{nk*#)%S!^Xon@*L$h@cJCKfT2fd9&YoCwor&2Bg=@V0^OI@28z8ho@WZU00Tnx|$X81^jzk69&}>LgRW^%wKi%s0N$ z_Vy_hbPNHW60Ul}AHpwJGHF-{XN5-i0VS z8mb-|0uYYhMCpUZ{#z5&4@8U$f}G|^2PU)(mq28scsTX|EmWs3q=Lzr(<*G$(`^m) zPSuZ#$Ck@Anjl(>^M8-tWls7;T#m<5B!~bLJ%JaBEKzA-Bmt-v=cSin+*Ao$uY{V{ zn5DxZxTs)CjS>NB8v&8+T>L^Y;mlNr=7!%M@>OT&VM z_dcypY+bv(9TsP-)9~7KPEJ|t1%!Vzjk2wrd}{<}qQN_B7h$NnQ9K90KQJd)`>l}h zpzPKY6d=JjC848Jq=G61Ahxhoe+f+1Eq$EW`45GSy^ba*RsN{=Zmf`h!UKYTAR)$l zHofB|2^ck_066w;tU#_&D5aFoBA9IW41*}45^9L*b1H~g@iNiNXKOI@irRM$gQ${C zk!PXAR)a%aH32|k!bt&0({8r5htxEGeA4ih5t^ZXmH>4T?Hx_r4SH$rkf3sgNS+MD z5a?VfQFHRL^R=_0`Pivo+zF7-R$yIkeAYq&I01U0vqr&eqyyoc8+G@cceZG?#qE8> z9jf)sM!qU|eVk!8B!>VDSg0wg@NDyXDkQ}umwyZ8L?sPD5G>JtFrKi3iUYa2pLW24 zsaoQA0*g%mrMhVlgh>LS-dIa{U5Ry5u}tGS3YfrRhF|mUgQ2q zi8OXtcaQ#Mu7NUUluZ+nnA!S|$+rRv_V7K|Dg_ydkV?CQ{Gjc>f%At6)a zMj>tqe}$6S=tJB!=e>wyf9dJ6AQF+<7EN`1D?+B}lZ>$FB{MUR{x!8yeD8xXPCGs(OJr(QJv-Uj?!8T5y)TsALS7ZgbT8q46LtH1@HP5!{ z2AbUb!^LJnR$npJnUi!_gV#2~H^tu67R&Q2FEq%gWv>`@Tl{y2fiA?y|4$Y{x0NV z13O{LCAqHQH?8Ge@K;H!#3$rA3Cz9?Y6es#N}<+|zC6Gnw{x{!KH0<+B?44@x0hxN zgd&bV6Uv6Ye?mGQufsPcXGut}E?`1UAf=ylj&muw@o1BwtZ!84%y#5)V=XrTm}XPv zdNKd(*v2w}ywT7@=1_zT3k?Y>jjP)oZ>!-0D|~B2_ph^-+alXB8X{tSD~2i|;XWju zV4oL5^gv^rf)5Xu+iqxAX{p-4PFN^IJr}aA!UU(mfap?JAUErr<|bk;;x1pX-`gJ0 z^nPukmsa`tNsqAfTlML8T?X8&ylLhUj8b$Lh;U+_9AMlf9`U0VnsrJ3R(hEKO*Q%1 zv3(~Zb+sF<&Z9&pZ`dRXKncLmC^@2Ei ziQrFrR_Ju>il&w$7a2L=rtDHA-?gcuq_a}tn^V+GFauJk05Ux9g++`O@bcTKiaBBU zMquC0!jrjOk={4s(~gE3n(&!J5wGq?{P|(u*_lxK9^NG%Qq~=<$noVzu~ELSeZ!HA z)zmDVMNQuhqw%zg=;*Hk@avVkSG zwoJ0Ca@BAE6}^LjxNh>d2M8eN7sjxNQ1(U6_C3d`C=b(e?Hoi`#;Q;WDqM7JAa`M6 z1<=Hfw`3!l*be-&&GiIWIKwHQ-OHY{HIX|KQLh~w;Q1mu5y!%=Wyy%AFpYG+m{PzzBG-2KC_P8spv5k4NPU!!j3)D z;W*+v8lu6L1d{{N!o|(t;pm^Ib-cAM2X-(;XB66d2O{9N5B+jK@zgBV}l|CaLsv+Xt5u7qr^(m&~Tsj_=m-aMC>*Kh0Rw;#b33pAWjGPnDQcwc{X( zNUZzBkgc1w39?<|{n@SU1oJsDUo?K*SVge#$Ts@X8nC-GOy!Rdn+hjRm;yD#Qlx}wvlJ$>l}@9FV8_xvjZ_{(z^ zx{Xt`jPy~5p7s;911H+|wcmQx$xsUGLVOxLf<0Z#jNu6v{QqAynIVlyzs-m%&@L`i{GzFIQhZjN>0JqX{m4)6E?; zOZgbuG&86Zb35n&b;~TG;nnD|;knan3i*d*%kn|3Jt2i6wn#!$Bt49nPZ@+j{z5KK+FK+7KgTRpf#+DcFexLui#Y6P_>F`DQ zI>T2!*WDeqyb5~xYSW?lz^%ttZ+RSB-0rdFlWUu{q<+=$8V!9zbQF5XJq)O==fyo@v9dMMcdXc zWahnJbLidOmx~V#KR&vC`1+QQpKrZCLtNB*`SRqUmgL6hfwHe3&%L;Bx%N}%u{qm6 zFCYHydwyJ@_P0X)#>XkcFXvz%Dh%Gt9=f#ufJoGyG*+1Gw z-+Z=x^FQ(J$e~5cb*srYzD1t+9{czAZAL#zroJ9q`(x+dKMt5u|3CI<9MA>m!NUOn ztpt=9{Q$8@%0p{f$Bl_whl`1(>s7tRZ8ys(j%hPv=h|;oQGH7F1N+Rpuohvxaku+A z?;fQm+-VLx-*x|(+n)C`w`Em#giMz5fc6jlaUl&}%Fu>cllJ=4lubw>On>`0M4&|dp7 zL=uMy!`4YNr#s(j9DUvRj?`q-^G6#QbX)ePNvH6Ag*w1-G086(5wSzcJk!%87j+^h0Qh-By>N1=U1Vx zH0$wd@2z{kK5C<{zt_yQ_3!@*{fJ%k?kXf(d$CLNgiRNI9r+;ka8~=+?|Ky z*>1S}?&8Nw8%+=xY_qCtnFc=Khm-XT>S|CfOj&CkGbFnKnD)YdLnaO9exYcn{-}W~NUWUwM*w zbM%}|<3+#Uv0J5fSKjWtf7Z_Ye1z-n#MorL;W7>FwSRmZH?H_*GyXv9H`(dHdeQg?Fu7TM#b-+0%`LUqKp{rvgtRV5XBbob5?Lyp(Vm7WI&k`Hd$nHSm8e#9ZU z<5GCiMCZzlm)~qFGCca>*sTW<7pH1It^C-p_btz7;ADR2`}XG>A)kh77Y_K}wA|$J zY1c`+|Gvsr`Ck31{{8I7Pq^s6^%a9!>#lvA8aBNC>5k>YtFI>g|5`iq_uns9|9ofK zaq#$z^3ST9H>*yJe|dfE#Ohzwu_u6?iPxq>K2txIjV&YhsXlP34&6PVwe*L(ZuZ?O zv)`I8|FdwDlnOr6*s(a6_tFu-yCd?1^aXVDfVV|z4;SvPU56X`ztUCUxoQ)bw^ZD>SS6Af$iSH|~dQlz$7y_KNa!>2Osfz4@_ zrGutsxKjg6KkH4#2C5w|@9R06oC!gBbHjMsn3c_;j}^NT*->Px+pn$U<%DXm5pPvmSZQQlpS)Nm-*oKty)n0k5#|^hHrzA{AT_#K;D`Wh$%brL z(WSg^o@HH!yJ}QRCelr{N<@G3Uutb|x1TGg9}{=9{p;E2|6KYH_u8I`Y#O);2hV8z zOA4S|#1I0Tog27Qg0)}+1dgiF-7S`^EwpW7+8>6Rg>Z1jY8JuOtu3CW+Uzf7+av{u zX5&sL=QfbE*&+30`Fk&BZqtLP%BuT89ndH_HjH=@);uC#*Pnem`>Zdded(s{AudX6 z;S8A0Ps|Tp7WS?`Vu2MDtSq<2dBWwjC9xY)ydOc{m{pt(TRo4#WtPRw43WdAG-F<4 z-O5U^fNC3yfD%y&lG(-##CzZ7UsW3$<`65ux6m1@LjMtN;0lVS#`e}?yb^A_qDaCB z))RpiNE=vG073&Tj|HbcuaAhCyEYR5`Us#%NRXk|(0PF$-G+^2beKI1c*!8aMi-#_RgZHC@HWI``TMs4OwYpqIE97kAf3&)2%?DNZ?1&5EaMwLjcPN z5Gn@++Hv9ahk_$relGbLL%$B43H9K=g_^on=}Xk^E?djMzV)h4LsOXuB#kwc|5YuDzIAyjaHO4tV|%3}{kZsqo!X40 zGaa{5fm{;?)KubY*DV2>c1VEz3ciJ*q9|Fg9fnkN?_L2uj8beze(QV@SCVUI@yE9Y z8znW#U;vU(2{26ItLrM>Te)OXDKb?-k^%hM!A4{Yq{img*6+PFn5`pW=oOA5VIxij zYYsU>kEQo-)D_`h%6y=uS6(xpN0JAWxlCOR(H5*3b;V=@4DkIzSOZYuc=m*PVmwyO zvw<%Uo>26Ax|JqrmSE{cF8zvo#+naq^wCfRmB*2qfj@O?L!8}1PDNhsg}TqkozKY% zbV75~fg#XNq_T2*%#(GsO#>D|)d7AI$(LAL= z#dm&ioAx{0lZC>4D?wiRNV~oQM6JHliuZIf==HJO{mY8Pdb6!UOE*DP{|Ngp2^HRM|lW{)im1O^j<5K}cffj&?v(gt|*d*+(2WwZ}yXk7)cF z66gbj(s3~Uo(+!y41kj;;QCbV#&5f03n3&>?W!EM?6);QqLxa`Fk{4R{Uhmfi@7I z1^}&c{6PS)H;1mE0{I+Vz9>kY>Gx>}Fk$U9kU$N5AWjTe&qRWg%(NzK^;Dg{ZWlgM z9zLbahI)gr=)HDWs@gn#Z+_un%(?Z^Ne3BEm=HkiJ{vbGhHY?-WO;3tq12vn5SHv9 zxG~&`zBwioveT6?NZbu>5*)*8=9aIR#|SLE}N(+upaAX-T$jK~tM zoiDQiVGT&XAIS#~E(DV4i2E{pro^g%MYtg|t!2V%0=B{Z))(Ehy5LX5QlVrj^tOFE zDG+Wh@&BO(uw^Qb0w72ZDA~Aa5cb_ARd+MtHHdy{nfH~5I(RX|K>2TZv4kKX0wxOY z{;()`5zI$I|Mt3MPjPhiE2a%K9crBJofDZoC%o?%|Jh~Vm8d``25kp_OBAsRs;J#C zln@&Zv1Gsl5S#NK*R>)d=EI4Iu}x$;blEMy+ZVoyo=7axNWTr-VZuUQZCoW(>r+5A z6zWb-0Fn^-Sco>{!0(E85gE|f+*OVewR3V9NyMHt4&5M!>2WfPoFKMrsOfiZEDYEK z@)074k`2s4AlA8XLlIPu2tlf1RMgb18jwoj9t06`8391Gk-D5?^b184g*r(Db!0bb zGU5jIfLd-Sj0j3$HR;n~7ki)<;3mc-Kv%MD^EU_FiicW=puK_kmHRR&4CunU&^D_g z1Pd*>T^O6df60L(gy_u+{0-p{H5TlI9J`7R{B#R?4?QS-ia!vc-Y!Sp2h{r{X7cXr zW0Cl7heR6@cA8FjCPwv11RHL_FDY6Vg}7u<8M_m>L?<}W6RqlrHgrTHr`;_E83}AQ zy2zB~;QQF!RliQGY;{K5U=bEs_++74r<*s9g&Y#$jhfWL7y*}9cwaGUT0+?B4}k*c zImQkHMccjN@(oH8^b>L3qFinFn?sj4cqcJxp4xq00zf$k@D}2Qyz9EeDzyY%6+@U} zW8H}Ke$9i^Z0yQ})@g;>$$rnwWW~&9Ucpiyp(9R= zpkh4;)8bPbvXE0U^t4R9U2^Wb>M5@CHDKee%h2~Fghe(kjVu=}-BiIjFgKcz} zuq@Po0a$>IR^Y(HW7tLo;erBjN{)A?K+K4oVb0Z;%2CL_@UZgdgH1%tQyB^&JVuve z9xAR6h!LwU6n@V;@d^txiBG7GVi)uF;OSwi5zV&%G?WeTaax6CLKJLaKpO6V96K*Z zVmZ)^XBcmlt0N&C1uC%|%sikL$3a@Ak&C)s5!Bf5`-#=DyIWC`j32R|yRZ_b2XrxV&G@Lbld+w{FVPUFWp zNMBI3E^uMQ`rvznJ{j)1l8SEOAXQupKgLE+>x?SPR&dTt8;0Rxg zIUo!Qms}VGP;ap6T|!$Z6*H$$i((OpFKQ6g$kLa4-L zpHjx--ld_v+33yI!(_SIh6DHqGh=DD#An5@V0PEG56m#)=|K=F0MMpiYuCKM9l@#z z6llz9;e8@rrD0WY@HZH+0{^@M0M+`mmUeUGqYT$8hUIeb&qN(MjcU^zOqh6Rni$P? zz8_@(CgflG^cZtX1V8xP`&AOu`Xpo%9akiVSkNs;3YDYzYzTx zR1-A<7Q)@B9QZ*o?wklJEQ4$iV<%adG&*V*06T}l-C?O6QNYXf)GXw@t=oZ0F)m&K z{Ot}|D@2|H@s%>ve){v-a|4qswM{};5qt1XP|~xD=!Zlkkr8UGRHfgA=sN&z_RI`I z0TjvcJ&ML3jnkUqhDlIu69?WT!aZf7)J0HhqS{>++MS3TP^edy4eCb#H{4MOc}Oz^ zkhZ|oiAXcFXxI^o6TxrLF;8+N-$L`Q6M;DKLx@mVMFcn^{82>>3n}e$d`$`5c!OtP-^G{+;Fp zGq}PbppQyOlAM8p=tUWZe;WNo-f>zv4KW37|FbCVV(;u2?jHq9ARziHG5574MlByh zH3!0E_$Le}ECY6*fiq>K+-8hD=mt*UkO}wA*oxi-P2E$)RcGEqdn#d?%QY#Y;v)j} zaS$069hw5DUt+!U*dulVMIVwuM!2VphBSv%x@Xy zAsKMFe&*1=&o4$0Ih6^?MAiLdV&zDl4gB-BjeV?!Q=J)j(cnc4WP|9c<%J?V0BIEC zHnD()GDzP${5CnnPyw`p_-z1?W~8=90nk`L69d0N2)s}O*b9g=C4;l}yqDnLx^l=V z1x^%{q^VH9t706~!YtVGLrTE>`^;KV0pa?;O+^0es1Aib)gy%JDb)28FYa>SE+U*1 zga(RnXIOwg_)A>`QRnct4?x-&_-zVxnraxNh_FWv(Ul<|ve5etOBy3!1=+PX zg?8esu)IdO1vtdf5$NCE0hA&O;G3#%k8x z)YH*71vuh4O?Wl5Eagemvgt&5RInDVOBUA+Zn_p{(3In*dveRn{=Thrc4_CdZ5>-0 zuGh(c^W45umH)-Cz{l)6@b zOA6SNYz@2xGUXevY-7pELmsOScZ{Fbshr2EayC?8?uLNf_hS0cvQfj~l{F8qWX}Zj zAI{&*dUD-hdoga$m+biw_UcEE<5X{C(%S9R#`LJA-K$ka<1ScqVab~stI^al`y*Qm ztUw=mb>ixp_1hWnT{Tm)>yFIGM#)l>IbMtZ%hFrqYw{9OdDa|F&~wsDQ!p?jKtoe8vx9??r?(8XlQ zFcU)2^a3|cd%;Wyv$Khk*pbG$wU9&Zqi6^@)nBMA_|v$uNn48&4`7;p`a&^owPy-& zG>M42!Sy4l180#kr?Y_t%8hVz5NJqBmoa##H;Nqu;18DpBO5ZLy!B#?DMW)FAQDEa zEfvFXvTw4_N$+gvC6xj068lO|)b{C~QxLmrk6gn|EUbR#Bk40Xxa&p){mjRnRQC>$Ag#-DdjE-Z4py zriKc6LLB%d{^f?wPKyeZMG9!evE<}U9hO|XSb)%yygF(G7gqG(oWK1{7V)(9r?i#1 z{7UJl-Ml}w>sXF%%3tK9TPu{v9QMh+#|oB&ob&11Y0cavBD*iPGOe#RYXGc3U~*b1bWGoOvw0PY0!?g@%pnxjR+`gx4A>{mgFOKN_E`|K|e91n_(eTe%Gb=*r{uN#8mQK_Y{z3UYo>-2y@f zql?uBLme^Qrx@^4{@9;o=s9Z~9Qw;&{|}@plh{tenqUIxkp#*Eo@V;508|_O!h9Sb zie(53;F?#Cm(LqzQ2brGKwf5eMv~`yom{98qt5LvGI+;dm6JjhFP_w78)LLbr}>tS<fYZg9}jG>k{+_^=|2~UKy>OOfe=0$Sp7LXneYm%INF9wXW-EMOwD0PB>9w zwwTzs)45&Olnui5&8^@z^3c_fjy<#xmnM78LdGT@uZv3J;W<=DYHn2jAA3}sCrt0I zc+_D+$pSRl0I;9-^is^|>7*19K%_Bf-AUjPXA$Nuu?T4*>Z=bt6}OV44?K-1fp#+* z?*&u(&_sT+z@)LK2}CZv{woQ-h`vi}(0ves0XU4tgA%3*RxFEI#BL_H@KAq9YP)GL z%CP*IWBqs`ygMC=k&WEOiX4vj*zd@1H-}~Xfb|8V%r?!ZZaVlYOoWW+_Sq8i@1HLP z%UFYCU{M__}PC|A&{K6xq6Z}hzFLjQ{VG$8OB$Yw=rT!^w zRAP<{mlHLSKNWy9S&|9-DMzuUOJm1yB!D#2WnzQ=yT)2r*DwGv9Hs6|Cw)QF>&y;? zb@Oc=e5=Na+jVT*q!_d}&tS+4d}GC4P0XDfnh-+G>7&kkV2d)b*WXU?%DfxoIb^r{ z>nsGi-vKSX?zNNNSfLFwGAr_YVQSC4hS#0VMB5M{q!$Ce!K$|StnQ&KLO8=wqXX1f z5bG?9oaRUA7?$v## zfsTxyC5IP6k9zECOaJ2q?BEZGbY`N1b0C7e#>g-O30O4)N z-Zn{)9D5wzj0#0eKGl@n?uioIiIIr0c~5 z{c3j7)jF#UI&>h60z0Yyw^hi7Ls@kv<6f_fYzo;1tU$;aL0F0R92s;g5_MQA7(PHZ zYwOUm56}e`>vhzW$!jbuo>H>}>8#r6|4huT5r=Xd%pXB>=C!L9 z;8VM$gfO&R93}{ROBf#;vwLlZ^zw0skKF#wPD+n^SMBe)x*jXgvQ?=`i|}D6ycLcj zWtTM$EVcR1@clXRm+R!RiX9ZM)p3GRM$-e`houUV?Dw5;ilQL1;osn{cO=T>tlH4h znW8WVCNC-8VszP5rX8pZ@{^;hx;b`19)Y4wpB|fzQs7#trjm7bJVZo#RG~Cdvf{Gw zbxY;2luiIl%xui>97?nmv&%R2|EOllbIdK{rQ}7sCk7WAa^NA7@)hEzdbG6S)nZ}J zvZv!#2Z2wD4RN-bFt8RGk5f)!d#x}wU-Xe|Zr(zn?mM{6YSZce&NRbf<-=iRvsJcU@oci zI7Qanvz}A6hX;zXfYRX)Aau4chfNg|xe0E_I+`<4SceF>5cJMGXt3_E0NA&>0LrMx zEda4Fq%w_RkyLZUR{`L=5tUy+{aj#!;z;})0C3>D2fFr!{?FbonRl_S?IOELd#~{k z6#b|<^k}@4zbY3%rvL&vNJkg5+a&|AXyBDx*28X%`o(pWsyQgMt3xk}Lx=fx#Yg6M!1Yt6K`E!S@;?Ww87$3Wei zZ}(ItxToJ^?wVm{*yy%5IfStQgRv6bab34DWF#Hq_oXP2N+vB8r3mz~5=cUn*+;il z%sftu2>;8ywp_+v$x);Jrz5orkdn$SVKM&y^&dY4#0+s0Q;;TTAkv$sB-$hL<*;*X zFpgHdF7SE-k(;P$Wm+mm@_6$sXeRAgEzzf(2pMlHHSmJ%q6iEJ3$xrRs#rxD9)xQQ zIJ2pl(pcF8@G>P>B@3P_I#D%Nv=c1BQVZM^Ymk$MZ%B|KpadPf=1Dx*Aq-5Qo7`mq z)^gs}Nu8f1$T3P;0RX zA#&7M3JAyx^)uC?WGG*^O4>jnjS8w$gegmik09dsm8#h8V*i0E!Q{Dr|66Ui|6IdT zCD3hO5Oo=wccqAiSt~*~<(`Ppl#F75c6KRNhE)C|yF>K4n&u|`L?3$I+!w^Mp4kpK z7VvhCpQ#tN><|=dK5oyHHMO&fGTBh!5_V`MZ|4BQZad&6fek5IE<09gg_ndZAS-3I zAD0j%W*CzIWbT01jaI-dz~m!MaGY4o3Zg!ntOY<%mma{=jW;AiMpDsO8c(CO69JMF zmI_Je;de`o1S;qvLHbiHvP%Tc!6@edE*VrUM&zo2?j80%>bV7Mw?@=h*IFAbbnZ!v z03_#L#Jlt5kpmjE z0Ng@I90Nk8@FP5-Y+{iT%HYNTNWEMDuK=S&d`l3-3yKb-d50&DY-Sb5FBiuw!m9F) zc%xy7b=`PjeNsvYVXSUd-DSEyh*&C&i^A1>UI;c$iqY_U@e(>ZFEfw2+Nm-il!E5GxVY-TjyZTQNQd-^df7IP4TqiAO`+Y zYk~I_gjBZxNd+z#^Jx-Dq=27A4@A&-=~SeZ2+$TFPQR#R(u!5b#m0)FxFuAignNkr z^Aw)X8bi=Uyq?FEQg6GBD&8^$Zjf?6Lr|h2BbbPV#A>|X0^dhfQJB^zygN6e+S*cz z;|4k^2L?YhLJbz+@^GOhD6s-1P3@5{-s`+iL`6@+^i$yH9yKe^#22s2bU4!i_=F)$ zsL((ezw4s_pckYC@ZA6y1LT=F_Fo(#&e;zINgz%N#67PO znf7ZOUs9Xo00y$_{s9ru3;eAIxrqyWe=3wNMC{x}ZXyBEbfh)CH0WSS&ls?kf?N^M zvr~$O0m$_3BBucES_=QLmtXVx%`L#E?RstnFdFX&FY!}cnM-F z2&mH^m9_B;zJT5$J}bAtl87Ka0hFuH_{9Zm-@rnqb;H~uAp3#}O>?0eaxD^Z1x^7x zdN*S4F`f$p3M5`=slhw97sN>ludios+?D8mC^6tE@oppbJ=WiFrM=Apav>2o|3tN) zDo$T0EM3BC4Iuuw-&u3sT1hS7Q;M?&kiL{M|AyjCe-a}{Q7shi1L4N5lFgn2x*Azy z=e(v*QxTbCMv;^18>woeDi`KV@o&=Eo%V%0WhcHKY#fq6LzSHd7y|z9_~H~A zOz{DHn`gHbi_xV=Dz~o3)=m<9DX2`aH7?~8md&lqPKwQWeS8l$ort_qaqTm_iU}V7 z9fHuMU3&{SFKoyDHwFd`@UsF=FePY2R4U~kgzi9L-Wam|0}$C@c4e5Co>J97<0T3^ z7Q6~|YWrUEP*;W_+Zs@p$?yf^w|SsyW_smEIx@d|N8WMNF4jVqCUE?#2??C4Nq3%AvntInp-_M!e3B5z;T|yC15kp5s42X0!^ddq~ zL=coDv>+e>DM2g&5d&C)f&#Ws1wjo~z!n4)6g76R<>a^4UeA8^^J0&2#(8!A0^ZCq z$-M8+^}V_f863kcL>8OscU=wX&I7Dj(3=nHSF|yb$O!iY#GmzrF268v@m*~-{L50d z;(UHG|8*L*pl6`bTU5CIK4L5HbU7K$fBI%z$lM{~?IgqWiG`;Z8QxT~bobxc-=7MK z)8Syj3J`#0aA3d#(73AUj58WT$?Mf9N^r`H;9wD=({7(adxkhc;bK7C>}JZ?wD8Vk zuGRx6jL$?waPz$K8CpVkDz7k>3)K|DcT4g&P5RsUeT)ssN0Wj0a0Ef<3ndn$CdjUz zm>hplnaiwBOvX(B-XVxopO2oXr!WS?O%0LEHQprqK|Fb1-Dl{+mVTXWgX&`V=1f4B zb_ORc!qpfj(etBymCD*=Q|0#~+s=y2)b!biN(OSwF71@|oK{K*y` z=(drny9zBa)$!SNRM3TT#*_$_P-Cwk`g|)mDYSl`t=R8cO#@|#Zyih2UE{JApW?&N zlA5J!gn4Ttu)N%7D@$~4A*$0TbkD|VoOIq+iv-c)mPmOR4H6FPV|c_e{nk66r|vx4 z=h5cLW47|8E0Zc=Nqg6&E`D4Ds6|0g|0?pEl>P;Vf)Lc;j7UigWs*HLanH>NwD&!o z-$cPu3DZms@|)wQLSrohXq6D=X^TB3kJiDv<9{dI_?aN@8A*J`E$OB`Iob|jLt^Sp zIRrJPJP|y+$N5IeYRNNDpHh;r_J>pv#V||w0?haL%;^G+YWds9)=qBz;sd?(l@12a7*=)y^5W$yBv!58jr&{7ElrcX z4bU%LXE$^A<`o47?_>}YSSF@}Mz;;1!7j%lPkCPGq^`yE9Uuh+yjOQ6>LSS64(IVCo0!$P+^#1jDjd~0q^QE zoKtqOEvP7FgomnCPXma1Om&KXE>u-f#L!w|8B8cEWmXTr1e7ybN|H~}y}(E?7-BPo zrlE;=PgksMCpq6Z^!F1| z!wxKeXWf1LkZuKe_5sMwg;)duxkXrpY)Q3N;KE-eMbU7`H(w}+e#jvoxFK+#&_>hEDi}hinPdy7@--u zaTU~^t$bXYHxy}UI&2D|f_b@{yn|?2X)Rud)UGSUkXcN?S2e0l7G#q}=`0fY#qg9!IY~nsD^j2W9?~3ombHo6v+J2FsJBB3&sa z>OqR`BQZTFUi`Y7gD7ze}8|E?YZ~!25&2R`HU`8icWYG z`sLHDlRnw$4`hpJW9#_iLPsqi#|FP3{NcU{_U_OmWP$S5{o>=N z$KBB^sp_MxaKBiCa1v%=tIzhq0VP*xsrZnEK8g&X?_2+Mk_kf>V#%8+Jhx=C>Tdzj z{Ue2uhz|ADPvZ~%$(uZ-YV7gAN5$x8Xl(BL%TMTMx7C;yhN*mQiM3u)P|?`DlGwVZ zaP@2M_Nqw0EY81Lc5-zyjZH1l{w+RaNd*$ma0fNAEiaCkU$wk+XZ5e-i`LzrN*ta% zd*y{xiGvw*Jm3H8{Uu6Rfk8*08T~xPC8(-|Z0Q_(IwVkj=j$C})|&f3U{@b9WMA2j z7qFHATX)!v;?+^Kvwufb);EU@1tjfF@vOXAdf;tk7s2FfOX%LIZUc8!XtzaoeC00% zX!)$P&vWZrt7Pjjn^9-GJ1d_irv_{|_rt8I`NeNL(GP^#i?P6pEM+)`m)tS_<%GKZ zMFT6+g$JK^se)j)PO|=hhP&Y@hJ$Nz@FV?|S{WN^_xh?#y>oSTcxrU?4)sY~-lA-| z_~HB&M~&S7ILbD^yEsiRe>y7}EwH%ZG*dR3(-`=3+HCufTg|Q?BI_HQG9JZWIUt%k z5sVuj)@q#3n%&nne}N6NBF=UduW?!PzVqq9H6Oa)-B|PShpS)HtbNemtlFlM?Vsn* z*$wYs$lUkhcV^lHzNqZ1X707c>pO3*{qa0ox^`*0N%>0tB1E8{Br3wPZV5Ls`tJ)h6BCRb(_MH3zJ58y~LQ(K- zx<5ah=^0zgiNxE>7ATwc$yW;UjbR#J>G?v!iI15?aD=1uP`H>TO6{){4^Lg``OYva zH2JMZ%rhO(Q)p*Y?+N34Mi~4xT9N&!SQoqAWZhiWB5lQ&hJ}d-{rbDxOSb>kSL;W0 zmCkJ{jdM28c)%Z8d2u|P`ZZj0#?(6j4g>U-VQ?fM1uVk>I$#7l^gjfH`!ThU_sSXbIW-(iD|Cw#kQg55zwYtOqE?7_%m&xLtdUB7F4;Dv)nEOy+z zwrU#!?=9Ukcdh5!yNOFs?OkD4e|!}8mq@KAw~?@KXCAcexOe0F*YA@54}#tF72B{T z;lBj?{Hqf3km<~S1UrA`u=Dr%nQ{ukY?kApk~mx8W85-Z>FcmCdxYv`HdhrGlsH!% z64x?U6OpknS4%52d(DlhNqk+G(A@I+X!7NS*Y)%vvp2`mA1A&!p82}v&54}v3vW&` z5a#o|e3hj62DWkQd}HZ`<3v^(iBh#x+#l-w@* z9-^2orAm@LpLn(HsyLeR?)?>Up~Z);*_xdny62nQKJ+a9mtg<;c<0CKOOS*x_~VTF zk2kG?#MVOC5$>*DRNllQbJDLu(hu!*Ep+WyNhc4edGj%@5iyIOJY;h`C$>Y5yV|;1 zs&$cDjo4u`Y?n$uFoRx$sUr_q8sp%|rXpAD)k;a-EB#SVvCI?9SAr)>#_o2m0WZZ^3zTElYukJFJ%F&=K5ak$Zv5w}*RlkN-rt zZeBC{{mq5ypI@KukbWt(DE$p#%m3|^JBTVB1{C{lO10@s!uU&Yeg8TTB8pN?NBtsQ z>3z|X%0lBJ-AI`nL=-V!t%8QINN^R;l{4O0@Zf7r$x4SL(V88R?mleipR34uq5ji1 zq`sLhsz0p{*yR+1dza4qm>wu*PEXTRf;D=V4i`>^3uWdFL|l{TvDWBs%HeaY|I!a` zRT|D7d$XE!Sw3?mEDGuF*4?*K820jM7`z1Kx#mt&xpuq)RvT zx968l7{ut|cOPT=j2l&!HXtTHdzTRdD)gscLjE=k!UrlK`+gU_vF~2FkrhxD6X=7a zgtiC2t|+L9Ffe$EtRsAV%hEl@`)?trUG_i%xQNMwCby7h6T-}0NEe=+`W zwQ1FIHIwz$bq{l!O0nHe?r<+M1pP1uJSgtRcCl|O59ljuQe7=QY@oNRFNfhtu9ovW zpnA7_3lvT=?6kc`u;-y$65OGyX-`*UX6Z12n|TnAp^-s@>CPIgSau@9lY13|h!)n9 zTp|~7k$c0pKu1IL^39`*Wc6VWOS*i?Di87D#{ojMOn)}SdAbKZ&xa^UpuWE;CF_jN zA32|O$MH30uk6-Mxuc>4^E9Iw+G2uDsVIr7g33KSptrmlKyDN?T>N@Rd_%Y_=PuMc zHld{Eo{+QI+ToSF?7G1hGQv=tjndjeZQZz$KeSHATiRH+7@VugEDF858&k%1uTibP zXCH8%v|;NTrIa-6E&;Uu&0xQg&L_5~W1~DR^Z6HIQf-~veP=qUz+A%yUQt=4Z|h>E zn=_5gP&?J}xbI6l&(U&AI&14#f3NX(;_%<&H6_w{*qV;W(w@)LC!d^^EuRZe^eWs< zlE%DG3qy^{z8vLaPn_PhCv07TsTjA&@f56Vt9$yV|LJATGmuw+oC}K99aaVC9_=lX za}Xk50EI|r32r-IeCqCRB+~X1&QMK!_znDZ)-v~YlB+i{g!)>7;~-L%q9FbUUueH? zV`z#Ys?wnuz1xhpxiO4_%Lksj*^FLx7uf98EWt)Ah{&I~y z&IY!IqxJdw-ZPLv`^d?YMuMcO8C4VFOx8{}A2^YzcARgzzCY1#KF~l@|FpKAg^(5f z>7C@1neUGE5AIl%>}l5~Qjp@bq=WK$s#D_ru`^bj&{r9kI?`goO8T;Fu9;K$8j9U6%_$ zI|jasoH_yEG=!i-fw!U<>Rt=98Yc2%b{xsrQDqPm`8u#)fI^7;wW@(c2L2zu#sRBl z@+`R^Kt@iGky?~hFfw>efI>_BNrkx^=Wj*5^(Qjo(^;AVK_+b8RpgVx$B zK}}N(ngaL-`9dmkMwLK$32aofN0IETP6IsnXgM+b9tk5y&$r$R9TB0S&4ap8L) zP88^qE15WXzw%72v3|mc(D{uGp_UYp`d!Y0n0D_{>@EMy@>`I5{SXsg9O4Y1&x0ON z_7bN<1Wu4~d%r$F&dhy>%R|saU@1yiw>?ntsRvZ*v=Jl5e4}HHcz_KB_lNJhUzIOVMw$8W`& zNU4M?q~gMr`!PbmLI}}EQ`80Q(dPXqHuULN_K&3vcl2RfD6#sEWtZ}y9<)TeL5K&< z$8HdqBp$c~J5+gjrM&>UVe3vQmBIY6HyYvh=YHHH7W6wm{76XVQ;!`FJ0nV#{6a{;)cI$-16b}bQC2LQE1Tg@f7!!n@71JAA6;i(L2 zlA*?;dgIe9$`eRGz`B+QF%-&#Bvg}g%sIrj{{bK?sBSh2x+D$sRTF|{42UU_aE?}Y zCm-aYk9zaBhHX7zJh4Y_t1p6xcp*Z{(V(MZlwPLywgKQc1sf>z1WmlQ-Uo*O>`kKF z6Cq57qwg@F$2~zd9WQI9$q;Kv=B9WKZiY&@$UxPZ7CX@}PsjwTWmJZP-%7oVV3pm6 zA8Kn0purTx>eV)oii2_cGL#Flwg?H$oKT1uF(W{(3#C4nVB0gW?@8I0ka7{c{bT*` zzG1>533oi>Kp%hK2?xL9fZTJzPB(IdRDvoTA_#=Y=R)}Lt)P<_Yl)GQk=@=n)D#^N zVxE16=wkxN-JxwR2jm{g{@`#>H}ABP^ey5~AMo-PEQX42kH_gh!ri1IG{Oi&RIBeE zAdQ2Qkg{)b;yNA51?We_c%AhfNL|FZqo6Oc4zYK;_AtS8M55%Fv1*}FxCh}0CJUNSYzv17N^3AgnbUk zeG%dIdj$=MF*iwk4|2n}xB@ffsSIEj$?)Z_o!P(^3=NYVhN$|PG1l*#O=-FM8Z)ov zQ${6RxFs7Z!0whR`<<6mX7lj4cUZGg$SL6^*#tltCste7VEGoaJ-w7JB+wb>BxFLf z&{{kIX%@CE((z5?TF3)bp;zNYJ=A?e5K02?NYD!6h}6e`yad-E22BB6mju{67mTOD zSBC(h3BWE1u7HD{5aaK0(2bK@9XXhL4EzKcmnZkbq2u1?9dz`K=5lnXLjTWQsk&$vlP>q8& ze2uT_#@v%RUf`OV0NxsYRW=nO+_-8J8F#1zn?%F6@xZVU)Whs+B{axp>1o+Al9N-| zD1->+xdfO3$I_qrLhOKbMD~5y-jhZKcVzMwK+aG)qKZ)OmQagyY&->KvlDlV50xS2 zOftNj(jpL9o1(G7jEMeQ!3T^^H78`iT%oNZ+#MNBC*vQ8aKi%R-7~z_TM(IPmvXRA zNV_l$-PS+vdDV^Eo-jEj`cZyiXbx7aPRAs8~DP;x|D z&@Wql?m@yXiZKwHRSLa7o(w-vX}MYGj|EV-NJuR2a>5y4!*G$AFmll>U}ZIClmXG; z$S%*W=yqW?OW^IiJPpqAig1*R1U@RknG>&X;-MdkFi9N59szvwAYF#lJBf%$*%+^= zi}?g_m1%YfpuqqYg>?2dY#tf8hX_-Y-6@N>I~b5e0d7mDllok1q}^Z%Gb!xZh`)+lEZZYMg*CngDI@dV)mc7r|&% z>Fb|-mXv}^(}|_jVCO`ql?2E+0a`X%W}qYIdnfaEBT!2vlu79>C*G{40a!Z8Z;FW5 zQ$fzrQTEC{ePhl;G@(ravf_2xF*H)IT4gnLDXXcHH-8PLMF$0qFV zw^HLEr2bQ!gE$6axfP?^y4&)fxrZ7{%A<uChXVK~NZ}e#O9FpG zMfZ)=>kZOh5OJmg?q&wQQ2^pad!;0dZ2SQ(I;#6kZZnZkEdj1Bb;RxB-P<|>4a{k(fg(Os5u&n90vX{hwU<(UZhO_+(YbtwEJW*G2}6LhmL=j zdFKYbbT)4G_}&f|siqHvfsoKJS3~^W$1;^D&|{b_aQy4-v!125fM!5WXZ<#e2v?~E z;7GtR5kcwj>q&s<@%Q*V#g*%TfUg<5>SYjFM35@|j^4LP^JyZLKh$1Dszw&d z-w7@tpl1@JT^mSngl7F-MfC?tSGDyzSDvD}GFgO~gJzVvi+SV>D~sf_q_Zzhd$W0e zpZe1yF0nDU)c1@4hg~Aq)~e^8@jpE8XF#W=P1HC z^jP*#vUG@meNd*a_WSMS(6pksDK>0r!cNnsagk;;o7vSlJVyK55aXYc07nE4ZB@M3 z-XDPR80y9dg(f-~wQO^>UO%wQRWV#8A$TIfV?(h^ufqn^ zi6d&6`Pwc@qB79swyQg!auh^25{Qw=3XFLO0EQlQrLY!+qdX8tPGAfoR(!b&qcO&5 zP{Q_E8X;97OY9}bP&bAXoMatlIhpUqCz-G34Z{BR8do9d10*QM*eCC(t8*p~P!g<< z%|pEC;}ypTWdSVo_GDU~^|h7(sPgV6@urWh&yR`6VuK0#aN9AgvD2F9urX@($S>X@*IZa&W2 zbU5l5?FssC+Lp6$Q+s2=7q^J_t7499%?ORF-}^l@;Z$`-SW;W(_pszkotNw4EFZj9 zt-alSnq?=}^JH*Wi&@&i)SpH=DsL4Shf-3&w-uQ~#jlUKi`vZh2((x=9oRDZjfZLvbb2Ozx{Y_(92>AjuGvA->F0BAZ8kx>lM zx@cxnpDX<|#jQD%aqsF0C(H){(L1uJn0@wdaN@34Hif2a1zTgYc8q?rR>p+-o0Dmn zATcbX6r`VO3;aMU&kCIE5qOC!X5-jmmW;17GLe!jG1CX6h{r9A5wqQcM({>5;zXNQ zu^UtQRV?pFau3r)^FYGugxM5pi^ptE4GFceB?|y)j!ZfIr zNbxaSLc;n4hhk|^&4)4zOA_e=d{FZ!U*Y`4Dn#iVo*3rjsXu68SA%29h1rG(_DkF0APFXgaFKqUi7QljVG)N^ZQQ?R;D9T#`R zoF>(@P@H>u+aoCSo$9x}0*T}7g1$&AwWuOXwaZ64Pv6-UF@bV!{e)_AfV8B|uYa*x z;HaWBnEHu-8{M}7+8Oe-MM=%uR_jf>7x@fXC%|)CkhrchMyx9UBDEUH@WUzj>EA@7 zVG)2EQ?|aZ*~e?uki4E$1twN~ZRxJWCadV!_x+ipAQQ_X?cgJ!E14N{y4WHL$h zk?^S(caYV5d#m92DB>3_N(VmX8Nab&(Wy&P9%E==e5-i7bYeLDv{;-}_l9_DGjrVVKogLm5Ag;5o0TcvLzD2PpPJCd0* z4*MR0lA}mTP`xAuh98MFRREx7yh0<#?t=JyR!vkQ!&p$k^74lUbPB4FXDdc1n<&uS z{X!^x8n48sRoL+gTS&ZO`okz}qu=5`}=$i(xpK^;-^{NosshOBa7=4ErJUhNbFWZ3KLK$U*6RT>ozoXI-x zrY2Wopq@QFA95pF{bR^$V<&E^N6k4X_g)CfY!Hf(5mfhnf9+`MgJX;#D>g$5S(iGmXS zQ2|`b^UA)}6n7rqix(nE?s2)aV~{+jGM-uLou}q;Sm@xO=sY$hw5v?qS&9Ks6_BGg zho|YI&QpB7L}UGh$AKif_)HDIxmeh)*~S&FN+U`GbCcT~BTO3$*qgPwb|hzP1run& z<7$ExEqet9UqX)eo6!s9jPgPDuR$NfUc!SQmsssYFWdBab$A}TckMIjAjFUuS-fe< zuRu7YO88U5QsebkFG*k+GA?jR>||gSz{V>M4q39Bnz0au;@+Jjke_Uh~aNvn7Ld=?z@ZD4;Wi{g~%mjxa?Wt zE4lOnP!W(Sxrt=MQw5+_U3I|15jRt1ugI#6>PG8cF(6K?)n6DIiX|T5_9rm%Aj;SW zdc(KPRr8vw)|hLwE#kyHBWZ$!nMY8g!Cm}9HcCD~RA{SKj2zLcn$I70RV5SqM&}uJ zWQR^48H`7|IytEN))b9fJO-Q#i&1~x=2tM$vSy~V%%W~*7}g^JIoMBN2OO?-;u5;x znIy<+=ltE|f?SH~;hq+Qj#5QB`c>rls7OXQt%$LJx;G6io^ad3Ee~D;Jt-k&bQQ3G zauyAIgW=ejBC8;R2s;%Oah_QMcH-5*wmnWdvnX)*=tqCxRT}gQkwKW3W&csptC&MV z=8Xp6$QmYd-s4Fne455oq%ziu3jKsuP7I_wxx7Q(a)%SYf8yGHV!rIA;GGk%a*|ud zh3o+_mNe4q}ptE(V05kYSOzAhB+rour1qe7q_ z(~Y)Q0%c+EjQRYF%gkg>0bBtK&t!_(C(OwCHKIdTm(TQby&klp`@67d?4dGpi8O3P z;HI^eQ9en6W(xa=>q+;>khEbl4BL?`&U^d0APIn8M43MiXQ&GEw`DS4)}P6@ksD0_ z{Z@5YOllBD?KG(%Y8Xrw!to-&8&#B>a7inqV8u`5I$OZ3PEmVC`O`Znip)SyFurVQ z3R=iB4|P2(WSG#ESC3R6NB{~|KwpBWK4sWT3f)44vOA36WuzkI23vL#{*(dBVR((- zT-buJUaFA7xKQu~^9jhVHlz>LfC7a^$cP|lgut!l`SC8P_-q_mQ}bA9ogdXeqYarstc0Fk5r8>v5^ zA49}|{5AVq3T?#@neuMrLjKXIBBndPFky*#v8`}7@c{nZlO1MPLm4wopy7p`uW;eV!xZ~<>A=H>CQ?VMQCLZ!!Asrz*Oju6i>GvDu#N#FHELCc2Z;Y4CA(NV1C1o zz3EV-lck4juOtb;=M^M44UqI9(MvE;tvR&Bs-uLhXzEnUmOC^Kf^$$6G3)9Yr>iA} zMEOHyk~LT#>mN1+S>!@55z~oCYW=z4y&9(Jnw3k}3aJwyiXkJ4xYp5XMH`)lYLqkDzxp}c6hy|#u)&r*vGf~3I96O=!dWumCJXSW7K!wuy1=hph z+NB~<=PC82BjjP`s?6}$z6>dn4(s{`(d5b&4*4niKxYRp(R^Sd+dxF-l!`C6ipWr1 zf`w*+I}P2xKimG{{q~OqDmsRT5`bC)Tb7=w^BzXD$-9*iOT0{v-AI6ab*;TO4{IJr zY@uSvz%gO_(cRplxJ=ahtu=*_Dxg_W&9Wz#QO6bLm56wD;;Uz$qTHz(nLZ8U)2w6~ zvTeD*tjj^%MQe*J%A^(fZR4k@P2-Ltw}fC+*wAS48Yg0YL^fNGa$jg>zSCYaluPP& zV(#W>1XjT_spSMND~49Idr71I6}!4c=Qjft%`a@VE7&b^q94jjKpj)>48Twej1XD} zXk{BsQB9OK>Fv>Swo~2|^;JZ|{S!-p8RY!jt6u9KakEU3J51}TqlH;~)C%#n+=P4< zB|k+5z3W-YvJy=rB#QdV$N_FSt>!opl~w}%x`cF{tn!EPDdgt7H%7c2`!sX+8|aV%w+`)iK;c5J{rO5Hg6iWVkSlWUPBDs(q-$ciZ|Zf0P!Hg+2bH ziw5cQsb6;TxIF2|G4CI~yt3M#aoa3*{4L5@q}~{Uk<&-Lpvq!`(GixGeAMrNv9dYC`7OHadKmF-$*~xxgr5E?&>X zVobsMw5VI?im3JSw=#+qC(@#waiH-A=qg7Q8bpuZM_llf+Yh#2(X34>Te(PMu|`)* zPx|uRT`E1iV=UQy?v6mtLMEt11NHdWFQ{T^Wl!m2%9fvxKe_+DI(go?SdHzj<+G&n zsFw0^-V-OcuM4iheDf#&vSkfBt@qYnqyhz^C{Wb6Wpl^!sqpJ5)p0!kkOF<{9IEimjKZUt+)g&R}nL`xL@7#WSGS=>2mS<#Js*WLy7a ztI=;`DULOxjTOfA)12X4R(K{%Iy?`A3>TV7&KkMz`7~CpeqmQZE`bqHbC>ZUtFw-{ z>PqWIl|Q^6w$5&)ZjJ@6OPeb!rb8CLn4_8Y4hOSOsxW13I zBePt0fAx4)VJ|^t9)@42O<68VLLJ8Up!d0*UlwLE*45;9pL_HDXH4aZ?FSbN$1|Hr znYBf)8*-U35-6BC1;&-BCRUvKz7IE{pxaxaZ+3*m-Mi^W`sN>}%G9d#Kc(x>gySU2 z+h6$(ZpJmNE7er|y{)C)8S`6n8F^J#iT{|a(s5I3gmEL-mq5AVtD7^LAg|Km+xwEy z6$E(d|Ir`MQ3ZbsO%k2W_esb85==SpEoxv#{jW^jgRiU)CM{$!H`g$OGv91KIQMb; zn{_MO38t#S5t}lp3AF)C%mgRmpvhHA&w~8c4y<9h!6SY3cY+6`Covw#t3UOBbSw;Ov7;dFm%pz)|(T z+DfJZSB>NXd)hFy+HDY{Zfh?TKWg*Agx`!xV05@bn|%yc!0VZ>)}Ku?%h@)1(E>@3 zUQ03JUT*Ph3bIj)_$mGJlGewz(;6U8x9#ZIbtS-S;}46l4&O_+1D7Qx72e&iemf`C z8ozzm8hk0`{m+$)6BpwjoZt1!;ls0@eNRuYqXRorhIHQZM9j2+hikL!Qa}7&wdbzm zHaFd229ve;SY7Q5PZ;Q+zfP|5+Ifep(K_3GL2s^~*ngE3 zy=vL}c--mOmD(G_dt+bjR;Sk}?k!ROuFHb^JNO6ftVa0zhWR6Qc%RujyxmpKhT*j$ z$2sB2f%n}@`lU9voeauWP7P0a7#^2!XpoC)(`BxhRZZ)WkX18#UwayIHpv@h>FoCi z+9&p)mjSUQOKBW8nBqg}kC#lX*ce^+`>y$c)#_f$4_Jmtd|Ewff6$z}Tu;y(CDPnP z!F_KKZTkt!4XaEW3N-ecHWnKjl4=gE{%P7&v7x7dn=|!T2vg+69%UL9R>z*PYA9|< zq)cTQyQRM^p4;trZfbgiO?UIOo3fQaF^}ihV{zg7&z}~8Ud$THcE_;5mar?aHIFm1 zZv0$u+jyf#CBDNvJ*gn>P|~i7TaVu+)?GS3DQab@WUuI1XuZ1P>ieEwD|Wqjn{hVw zL866x)!R*@0i83euUfnPxLp_9?KzZG+2vihntTnv<~MmjextL^VEah@n_a58=H!mp z55KJIUP8t`UV6PfJ2?X3o%&R<%RcR?Rkm^PzXW@Z-Q88H+18zNH*j{XBZ>T-$?FuX zT5nChHG7qUdH37?q5sCUD<5rrXOVg@l8n3uQzuNCo31Hyre2=j!08EmfI zPGg_Qe+;&JiO#w2#Dxd^|7EbB{8JIf+YH^rNmtY3gxQhtQ~xs9zu%?i2f$ygxO=Vp z%NtRoak=01e;MqZFYjLeUj|!u^NnlEU*`X9uz&sd|Bb=+%Z!j2Z1nvT2NC9cPX^(? z47S7Dd6~gBUudcdN?JJ0jcZ+KKIY=dpy1QZ-`1+xht2c(+Y`fO!yRwmw4UvghPs{Y z2&n6A>3rS#?)1S@HT6 z?JLI&4F$=U_BnDOZ_kly7kA!sRAk>)SPT#`#v9HvZm+Y@!la9gDpq}?VBcNWbLPZQab4Nd^Pw% zckFyqLP-=5kSqqo6`G_}5ZzJ82x9J7H z-tCYN&m73Jr!#fcL;0&8)SYVIY5prbLQ(Y*n7OOHAVOZ>H>1n_P=q9Ko8?i(>uT$& zn`{nP#bMW{(woXLj;^$zS2MwcJb#iDba*YGbblR$yzwRg+ z#dV9m?c5xY3?Nb-d7G`~fLJGCNUm=`~Eu#a#+Vnl_7y z>vxBJECsg3;MN>ru6R{=T*)`80miN`lpRhJ_deCd-v6f;4%HgwkK2k^XkQWp!BvH4 z8ZZgsiUX&d6=dJaC0+^g!Lm^Fii4j@!fATVI_j@2->Z*4Gi<8vULS3Pf8}Sj`hIo! zu%~(LRK=yqEr(asv8+>iAH0HZ0rhzd$TQ>hXa9PT9-lSXKK4L(06Ij;4^w-e}IR%!4gt+cUmE)lm`11!O$hz zuxJ{);O)Z$h(L*){}?R>=sdI8mW1)Q?|?J}^5x~;PP~)u1<>sY{l>*3PM!)vOiKt8 za(j|F+*g%WJj;Vz7NaeAK@}J0cC6729S@vu0Q;?87%?lEz5@`|cCI_@@M8x!o2go0 zDpYi<2hzt8>8su`y`}BE^JQte z5MN8{-*BO4?7)ZnCb$HTx1U3u57tx`2P6&{EUgb>HhqG1t05IxK4UG{HboVzY&Uy8 zbmy55oKPE*Xt9@p^5_WYe3IT6+Ts(k>az8)GEKH*<{vek$DM`^7&PY;d)v#@^zh3x zzv|88zj<)FY@}bqrr`4P4Q1y3z`xZrRC+$8-pwFE{1E3esC1m&7oyC_^OHpKAN;~Y zm56u!1n!zcVl&;$nY(Cf*;8?_!{>7H)9-0_P2wr}7T-dnO8(8J%`~%Sr|Z7y2CR>j zZ7V~9KENd`=N~qWocP=hx=0XYXEB(JbBDj3on2C*6vaFB0T%OTU_(#pvOn7}*Z(ad z2_zBO85Fr$`T$I0>ZTcH7SSaf%)HZ$vJ`etlA9*h88fikxj@$P%$ubgeGpB)Xjgs> zk2_I;!QpC8cKL5lO1;Dm6tgsrGK8>0-oRR#fg*gVZ(YDAx%9Abl@2wgTq5_>T7OT+ z-}XsaEu6A&OTT=H6mm#!qFo#xEr1jooDI;-x&9BEzP9VTj(pGkT*GL^()VMu@oS^G zf$o~ER!PG*ee)VVe=5}t-p-gjt;t;Tc*w>27HtjfPzu+L4@uQ@v`W67kLq~9_>9KV z#>)bY>mZg(gU+`>xlF-~?u?peQn~Mwf)$pI&;BAUU!N#TD|J^Wt-k$Z!^ryf6H|JG zT{-cMpGvs?{>pOzH1~O2yo%_$V)T28!GtSLMPkEttJ>h$%7jvWGz-@5tMv#jKHu{= zpup=K!}7)A-D9fP7jsP5*ZiM@`eNV_fAgAFsZPjA(|c78aTKiy|98RV;zvUq&-jyN zOm!Mm_1a&glx$DDPf--Y`qhS@PQ*QNn=A6_ET=q+D$pn{gBQIwQ!jwV8u&DJZ1A6s zGyZ@OG;jwL$+!{@pzOGf7hqofFnOCjxL<-hz@r~c_fxKQ`jZ~yj4qv_&WaIMo3k_I_3?~oS+Qimhgp3yXs88jh8IIvkpqXiR8ivI?C z26Xv|)=WQO&C6%b|Kp}U=xqmj3GYl!awPVNk92YQH znFsCfC{xO}P`(895^Zq(nl1~kIZZqFzq=AHcnHW6$1li8 zIE|rJ4L(r3} zs()xSEnt<5Ps?bu08Cx>!4pAELG`}%K=ij{DFrYj<7VP(jVTaW*4s|DsZJvI<42e_ z0}RWm{c~%5z{r1Ov;cfo1C}R3jA@L{Eo)?A+J#g@6#Chz{cjvSyU+ew@4s*~$NSdl zAnpHwqa_v51*j)fISIYE`AwuuM$i3AMn9)xze;_e&2;<>?Ql7;v20G!LRHx#GjI(Z z9z#AELnb;&PN2rYZXRAnqj%i$hR86gOy$pQkWo?yZPORB+@esXgXL;D8=?o{*%Q1z zYFueoQE9BQwg@c|;Uzr08+FI=Pbfc?Bo-ZCPEOEYz`jBML!(a>qNb>b2nlwE*hmZo zM*z89P((`HeiNal2S-Nccz47IpgM9<2#cZPy_0Y`55YSmTs$4WCmy#?oS{sCU7_qF zzYSi@3rs|n$bH3b&PdT|j*+#R-()pF9;0SxFqwVZ>y7i^0UlJm3`Z-`EkbBGherE29~s1?hX-|>VrEk z!AeAD?k&spg!qml{-M#AmgJ@c$h^})GY=mRAjZYKn}T>55PT%!P057z@Mam1{zh+l zQ-@tOw?S4BR}vy`g`JNYgf#K+8Nv86I^^Iy@67^+)B^nDqXBHXBcYjmdX;R;H%88U z^;%^S$}giB7L9VThx{8ye~$~-CUWl)F**#0kb#wB_)o0_UC6jD0EA0L03bq42vChO z1uenVWYsFtp)C^Jgm`Bny%sM4+r^h2Qt(p2B^ast00UpmL%aOK)d9(hbo(B8PzI#^ zMnHrRye8>L_?M3Ev;!lB*a=#PtO33aOH~&E{|3^;|A)2v{Ay~^<3GQ5dQYf^YADhK zjEEW$H54I$XeiP|4MmNJ7!WaF+X;|_CP6^~K?BkRS&ODh1v)23xvLYcnuh09-E31IkJ^8&gAly=%dw2+31DuCRc83;)VTgm7B>#Vb z^jQVC_92kV=*+!{`|8i_&^uBIbw1w!1ViNC;aOg?SyK#>hZ7x!kR039hOK33y+1OD$8NnYyzmq_dW zHPZb`?17!M6TOHDMpCDMys8mM<-+Xoz(<;iS2(6wMd?&x#XQ^!g*+Ch-o!xdQtdZf z6fwf21RP#!1fh2+vE3?40jplSnDX05Gl;jwa9`396Ipm~4W^$u!1aLt~l0D`7Nad_(RI5-RbG~c(}9-ex_;g5Q9<; zAvZDeW?ib&3E&r0w8hJlA)Fk0c!Oe27O++03UUkzGr=Z!$8IQM79E--9Qo?^^94q$ zB~GW|wHg9CANJ9>-t^`xiWPNS3hap&GjQXy!S^~N>i#3CBV_Zh(39TE>z*5AYJaM3N&qe`(1Q7y0`J?lP6zNUlQ z+FJBkcnhUIY^FYwt3hstZ9flD{1AxvkCBG)vl=y;3-a-#1;5bVr?9ihhYe_*Fk3Z+ zj|cJK9o5~lOxy|!l$!-+`8TwK9CJ-c3g!XlyZsw3lG@mN-<0Ac%SNlALP z09*qDX$8ig2_GII1*7W_Q0jV z831wi(AC$0Msmr!M+=yWkWzvr)SHmB-FS zek*D=;g0jNNCM1bnu)kP(E{EADE+q@TjjjgvBM8B^>?d(720t63>kScl3CnbO8Jhnx<+FTX+0M(}}uh zfb<@RXRm{A<5a@C6{Uy{-Tj}MKFi$eq$FI^qD^mUYXQ)ViyLL)(8_jLO|Pj2V3n=n zegghHy@nP_FT3}YIsW_}V5dYoDL_*iW=-_@2NOQO;j}8%ReUL|gxpW_eY_G>NR(G! zV?HS6`BJDC$^-da>Q$B5WFb!F3!Zt?Xs=m$cZsAaJ!u1<_9eLf0bA)S9~5;YeY{pX z#eDY@K2)ow=sk=J3xOwJ(Yt+-FXW_uPN^hiF(e*AvWoxpE(t!rfp-Ed-%OJ1fuIRBiHxV|`b~a(E$+q(C!)9tZVjR*Nb4HH zW!Ly%2^b^8OQWEFdx{ZKv3V!QtB*3ZJI)y-9&NEQbYo7HKeaBP%q=VmQ)%kF2xB;e z%Ot__P`)9ZIo$}M7yE|vcfIlRxj$bLO-y<5y%?6#MkdUk17}P;T`@D=Xf>M1g-o53 zCb^-A10{~o36Do9#>m2$i~r0cG;{}-5uBI`x7JEpr3FFX)`NxKopmg=ED~1}lup?UZM1P4|(S=mTIV?zqcz+@Z z^OYkggaMPoASHWe^0L3sD;ebE?^*?f<*MwAa7-R9+wN#|9};I*t`K1s2)mC2dU9)) zykDU3GXXqhimRIhV?cHVD|TZUX=^vzAc?D07;W*KZZzKTCd{sUA6E^C_ZrR) z1DNmbLhGss_fds>Q|Qk>{)N&at)1*&D9!UdWl9U_-N#8U_P|Ut?I@ic2FXk|(kPwl zO`QLUtMfHjogU(6w&T;t@*|ri^SAD8YpPn~#6ntzqUbG1=6LE0=K7a$HP)cY!G1=c z*qnHVi7?S<`dHe5Xql){o{Bpzc54oDlW;`X z;>&KvEUo?T*E97l zY_?;ZDg9a(P}2J>?neudkR>NlKj9I?8>kJ;?N zC>mM32By z_0Xe6G7x%2{N7oXZP}~V`bMTej}?+!Y|!&GgCh<&n?ZF+OvZAf4{)fw@Ig_UVq1)s z-P?U>X9a;@uqOIj1a=s{qEg^lhaBg`EC>2c@1616yaEoK3utq$`yMxG)E5D^Ba{H9 zZ|ZLDENe?E+)|(=lYL@v^kygqEvU6@S#xwQ%jcFKqnjq6J${<#buhu@W#S)ZXwNTD zNs{uoUNN)RX37z_HNO?PAfYSH)fzAjZ zEOCU8Z3-*V)F%~gBMwQ^^tpuskbRVoNgfM9_|EXg4dMY@nlE+!bB5746X8nI!`S_{ z(zO8j%)zRRD4fX#_I%goTsj^S8FCW>7v`&sk6LN)Co#wxl^;RkOSZWu4lq^pFT)gq zuJK4ZRmgU}BtjTX@5J1l)JJ*i?T;Z@>uu-w%=|>dm^Gg$+#2}I;MDA<`d_-aCL+%? zo9ds54*t=@VN37Y7#%0aMJ1ZIB7N+4^+EUc*E)AX{sBInCt|O^)YA%k<6d?{2EKM4 zQ@pW&Jn>^OZim4Y2g#i2se*ma4=tvK45kFmQc$}W=*u7c=aEW?L@LQJbo@<8T z+%Y6~TJkQGkRCYYCJeFg=qo4OQ5$DP?T>Kvxnf+DG|2|+zxghvd6ldwwIMWJdDOP_ z^no_>>-GyKWaE~4$Q{UE; zz_1OA@=|{mA^8&&MISGZ_RMYh8Y+!aKpiC`@=RRyCv+}pozkI_}0n&-~pchnmK>d{&KfB zIWsoX+E#ltcFH>~<3+v?94c@yL1uY*%A_lsW@8|9 zM!A>(AXD(Of90JU<9WFA4yowG`V!bpi~HbUrUN#V3#@{Vaiq#lsq*&GsvM&TnhWR* zr4bx3=Bu#>!T>KGc5YaaKP&q>P@xkd&F?F5@z)A{W@DL?_yFGNQwiYGa_3{vg3QIT z&9e&<`I2}&plmF=jZX1#s3Hh+9GmL(CKq@eAZ<4j?PG~HAIC?lF;@TO(@zxmrf2T$ zTqu@fSH#XvgV2S)!@xMSri&jZbJQ443>suAOCNclyx2?7IA}2{XCGI*kAXoqRo;6e z(dQ5<2Ql96<^+dEhbCF*4=EsQKBo_EJPsW!P)XRxNY;X$1Tt%j1)tDZ(oGnTBJ7Mm z-N%I-)si)=vW?S6lSlK@H0&)_mk5G0A-LHf?np=!4m~YO7F4f$dSONrakK*A!xN=8 z0}r3g*F6#=7Z?=qh=7M28#!4Pq>`?m?ALQRaf_RKvp$^s)wLBeyLqpYIOj zTpP+wk`W6AbA{}jH&OowQNPn(J;kDuiOHDpS7DrZ51{>>PCLJYgzRSxe#brY-Z?WZupoB zRRc(6JnSN+_Pru(ArcnU)$}3dM0Jt_(iQG4Gfz13q={^#ISb``M0~UfO41biPv+p> zp{oAFr)iiX52OzZX-v;co5X~&a}s4P`cuG`T`22v#BqMhbz%M*mT2I%Y@-aTIRR~W zjj`6a#Iw!K72;SrTw|OH25YRpsO!gX+ens~0o?s27Pj*V z7JNyv21e9~8(zqK!Tuem4fl)CXeIE-=};^4zmA%Dv?Jcbi6(tCVtY!Hokhd2%5TDw z+Ei7TKbq&BlkLwkx(*>Tru!y3s`l~vk^!VMGxz8gj0MMIp$z_-2NlrG^_0z9ST5Bq zVR)fgMUwxcsSy_Cs8o2boRyQ&5p2!OO`04+Ymfa$Q`fpPTID2!A{(w#e_>lmWS%_1 zzm!^qsVaipX|*=f;?)3V3ttISQu`>(hVpE99jO)4l6gx8?*PfN-09HaxIxdE-xpMr zix$F&nxDNus6^pzFtwsZQIhdHfJZla9W1ETw5cfo7N=wm^hpgEvh^{czY#SIr@h%r zM++enl_ZY!Xwfuw%k*`W5bhAi%u$LK(J`^?V3<*>@1e|tP)qpPMm(s@qORh`ncs}M zN%YUb=s&C0j-Oil#GTzIlc?o|&k`XWjHrb?Eas8uP;yS{WW{=6PT(RuMh&iJJrSqo z<|-gJO~p}@J89+E%Phw+i$Y#VS1T7VcfiI^o@EgU$9`Zv8T9vykP71LhjVkLqs=y7 zo9Q};$-q5X&*L7Ogic2tIhu#f5QO=1aKKBGpdQhqd}IP}L1KXo&^pSppCmK-;lE%NEnQ-VrJFb4NBnkco!a-UPPw&G&N#dhopJt~zV2_Sq zMGn%H>LSZMq7#b>-+q(M<4ca_EwH$oFT8kk;dS^oE~5L5>3X5B1jtSnmL+Qt3pM3_ zDxB}6#C)a1k#Dx84xux@zW#k4f)FJl39|VxpO#_Z^8kP8<-DXlNs~`xmP=D{l@!4l znv3+T2pf49%;clCoRO`8*i-4arF=>8r$dIza_3dUVo z?%ys;5?B!E*f81J!2?yb@DF<9zd3Pu<<5#c-E}dx5BI`_UZxRSm)4 z9b*WlsHp`a4d!fu?-kI{r0>{R2OI_ZZL0I0jN5Au;Gy9S(~3B)c*q4Hhl-ZMHe2&1 zTqwS>H9Kyak^C>HPKH5s6!eb!7&*L*u_vc}F!&E zw}Ezn3(eU*YUO-IR#$_}^!WupI6SQiC2jWywQ;@zcsZm?7Dn#Z`BlqCf2`|JlnQ_qU?>`N?28RdVV6fYK1_;NTEob z&Hfy@dgbuGC}N4Oz-7WmDc;x;iZMX!Og8vAh@mexuo(tt za=^znB{;6gEEIUJUPAr`+41BwK2$+Bk?=*ZxHbgVn0EsdHDJb{^S$FCwL`=NJ}v%s zkIY{Rr>sfia}0&cC>)V_)aQL1aMm0C)Go1o6V2*_s9@y|LoGN{2!RPS115<18zKxT z^c?+lcZ$$#R16F03t;;x?%<@B{x{WbR~p+W9;f7)Wo+OyKEFwI#ugM(Q{gUWV%`2bMO$3Boa zo;7P}6j|2t0bF2lmd!YDl|*e3>Gs{uz84@%iVy#H+#UbN{K* zcS_r**d}KfFb*{#Wh4_*o`dY^pUVgY$3*5Rqo!L*wX*(c&-Ce|7#T;bh@x5fi_nV% zETLWkQbL%mJR=i{yK)%1t;v&IUN;6swfxz5+RLZPX-9YZ^gOGeLqtc$6r@Cy1a)c( zITy~@+&>v}KJDF`&$CCB@dF23wtR6OKa=#Jee2sVTgM0%g2LweH%5As^T@<)>=XK4GdEEy~_q>0(^+T=p{@yjAy~BUkJ*AZ9 ztvUZ@VZtW&&S~>kOTT-5|Mcd;X}BVOIU~nDaW6LS=aPSZejj?W?#-n=i3jHHpd;}{ zqb3Po-!(pg%QfL;;sm|)9+Xb$=M^N~QCIEv&*y&gcw*}0+C8iA{ss||=o!_L8(RmT zhMg}#A1_Qed#EhTIo0oAy3S?VM*iDi$GhmiUY+Cb)MCJ3c*R&f{$X zrW+Ht+r;Qz+sLfG_gNLhf0xBeHh$=@j|%PYfF#mc-)yUQjAz;(&3w~oe{7GF$%eeG zaUCJc=T~Oh6-yRufUHj*d95O_1&?l1{1x$Q88;gvNkzu zx~BHekjN9w=D}I7BN(k?_fBo-s@$Nc{hz_^yRsa<4-@j>*w#biuQoQvUTA*W>Ud$n z>#7suGi+LG77-)P-m*?|?tD@Cufc9PaVjXcVryp0-Xnir;dqZ9RPSe_d+ML8sqc>Y-yR|J$XGI-T@* z_-xCA|Fb8ji`G33Au!-In^j{MlJic9W`(G2CwfWV3 z*hE_jvnJk-DK~4L+_<>28-rLpZMpJN$A{bJi+^{if1BvzLC608Fwq@P3@LsNPjd_- z_)l|pfYPTDzqyJMDQ?z))#h2W-aeVb%XB~gV%zj*%UDd?!DVObo9K)MFUt?vVpBtD z#hINo6LMeE4S}2{!!zC9-9~Vi`c!(B$%rbm6rABpBZ4^sC87Y;x2znN8fKh)OD_PX{k%w$~{4M zw-%50ALzixuQ9&hxi$t2*O=aaCwzZ@%eKoO*43wYYc2ayuH<)Zd4J^X(Cu=yC3TqnO87Cr^;Ti(Nq60)5jFM&Y0f#%If?Dpy&@kUyG@<@Sl)-X zxWT<0wW;p&pw+?~XT8IARc|^N)wbl=L(8(5Bjb^H|F6rYvvBd3!)>R= zHD^9t3kV~9yo5}>{QgfH~=}*Y7%LmFO1IWz~@{hD_S@6h+Mc{<`z+ zx(lyaI0#w(poAoNAO39dY9M(=%aV{{#&kj`dP?eoVKR~pD(K**+ZBs;xUwv!tEWiz zvoD^yE`HH`Y~avs@Q2BDvH9!kctf>_?2!W&H`ffv&O^`Uw{D-MVqq(jwjVybD*Vbk&BXM5lajc1 zYahbjl$H-JxgL%G{qP?H2;exUUCwMKfFaR9FUt{efNcf2CPyC)UuBeOb7_3ed}8QR z<8E5N3lpbLbYtwtR++r>&^8f1yA4 zbBXyYbM?&EVNtXy!u|Hm##1@W45r_uJuT_gHn}j8(gl`s*g*JXuZc515Dme+m%}!Q zFTqpX+r&kWyq#}DqVF!itmcE&7vA(idZn$U)59w3(5Fb(l(3Dlr8%~fqaZ@1C@{@2 z{Ur3aL7rh4K@2a9sk(cC@w@m7j~w+w+t3I#^|<>lDMBgFkJ5M>I+d%s4CNqO+fAUA zBVBJp%Ly!fa6JVgZ*&UtcR^wDc{Gh%#;pSzy|(z~<; zoIjZByiNrbWLO)DYDX*0#pG?S= zkGnsua>;NwQG<4*0geMaYG%SAupo zOg&m<2;r_pXs&&6|IA)70NB7Or2EK^&W>!Pv34}?q3@p*qw~S%&Kj=^AhFFOTp;66 zE7Ni4Eo$E(Htx!ohb3>#a=O&yu)0IwF8ImUC|JBhXJZ=9dPBG-y>gSW?Jj}sv*680 zQR-bAlPCo=|HBsAbkY~AwWwL>2pZsAisY_a3z83}lIt<67{|-~=M_Sy9Al9)B?`p_ z-$XAE0xdgrN@dI2uS3TiJ2Y<|Hr?>GAKq?^>e4kyoSwb$|B-cM&z$qAzhF)7@2I(a zw>?Vz{%xDN6B{+e$55P@Vv6uCbwHL4xW>TA+%bz9fNjwoQh-p0to3)zS@=>8Sb2nO zwnp64b7b~Xoa5+m8tCCX8bBf?(EZ^e(X-KBBT5N!U!)i6+&##Sl>KfjxMPZaqJPeO z6{>^oM_@2Tvxe6eG4E9iKUGY|qaHe(ZNZ*DZ zL_6KPqlozAb?%-|#G8Nh6R@YeaX)1{p5RD?4C zx#jFxvS9_Q7jS0o=#+SZ0Oo~;@M4D-frf?U+LwczSBYNpadFnSP2BH_=a**xvcr|y z;XkuUD1X$oT$dN|9>}l$fgM|Xl(q*NUFRzI<=+Pv4T=mMQZ3}9ad}MoR|lnnJf+61 z`ig_G>Td43z|+fu|LtcNE2IW>J})+XG4}i|fedRVwU38Ni7JQq!kG9_y;meko zo?iCe2jwmZ{erQXs}D8ILr?JtAP+nmzM}EYqLL=*-d=<|bKTsp{-6pm{MF!^KPm?~ zziYeLusq*d8T;Ot-1XTNxJN1%#4R-d@-*Z!0Pt5(;$|Q)IUAJ=VG0IrwRfUJppJ&} zf(OQej39TU9S>m+Eytv1!kO+>Fd&xir!zq(fHcfQQnje1G%)HDH@pnkucg2%KFU1+ zuYd_MmZ|`0DiEbwZUU=f9%7yfe9c*1_-ZZy@R6M$O^w2{ODw-pT@?Eha()+??K=0< z1^b<}z-A3O7XXcREPvUA$kHv`sbhj@=B^`+VWZx_g|8^88hclcaioEV=6TK05L#7n zN1OHV2{w$caXbAQ&{C z)71m89d2geCTLVRCf)VI7sktP&t)GN-zA&RQO8)n_C6sko(|lpz>Vww(Ag&joRHtYK zB`?kt)_5Fq3O4N<3UT$6UZ3@KjO7pY;RZ|Gi5P7~kfp4Vk0ARodP)rhTKIw^uEIbTE`2rEJ zAr`b=qG2$~Fs70;DyQ5RwBAyp#id9Ujg(D;lg*T;PEMZpfaN(w3jolH4eXbapYq6v zxxX9DwQgXRlJqA87oQ1E6(x(mp>6>rv=;tv(pCwOJ}K~U#P+$IkoBMl9&Y^khFr@- zNZi3(P3tI)kf&pl;jrj4I%@Yz@~D=&h6$P|mZ(aAr4^g72|$HrKU$7~E2L%`?)%dK zQiUB-Ve`JMM2JIR>6a$MHo!S@2CiL2NmT*xV&a2>;LAlFx{r6S0~%CiVFJu~CKdCN zvx!R?*T7L7+&GWq&W-b^13Fa{MMfxg#8v>29>OkJNk)X6Xsn~L`8W%>a=_yXvQl|eBEaurZ-$grB;aDPP9r*J1{`2e`B;myb^O4n1+UJD8w{_W`!q8>( zoX{^zt9uZj7WYL?j9{I=%GhkECH4x4V~)7v8qBy4iSa%CRIUrC#+%d8Q5qu5lX++M zXayKMQ(zaDN;CDF>xA=VqOYjoStD@c6ES5Nv57_=W1>;qwVMXBN4t-2DJSm__it8^ zVKE=V#b_a1L{$vDS4g2@TWRnn23ri_HpxYhn$k`qCoCq2nP&_NsaJWqbxdj}qpOHZ zy`jWLWfBF@R$qlwdl&3x))y<0#X1$GUm5v1frRm(DAl+I?n(;VvljBx#@lODfKLAg_kT5ll6QFV-atEUr4vu#SC}j%Fb~T16SndQ7 z@6t$lawH@nhUEJC+Yz=z0uzJFH*<-%S-3q+td|;~13(s=(xyfG^KP{%5Gnp3T@D24 zj$qJo|D{?u^Nc-3qjV`Sr#IcfD>BZiZeRI&+guK87u;HCZ$`R{Q)3?5DuaP0%pVc z{*@`_>^XAsJ!Ry<)#M2%#zIc~KwGhqkNc>`AGxqhQqXbU$kASfEWd$dGidG*@2HIS z#fCJ_MuA*jI2&~3ZvI=_IWq%y+^o9h6gf-~>Y9jx^YXfn2zdLnj0YMCDpeadh9?Hr zBZtnFH}c_?05fRkV`@Jw)>Tc)Ub_xt5T^lLNx+uRikk84?Fb!!GoOavit@i8<2(eU z-qGyy49Z`*KIb)$ns&GmXy^$pIg*)vPfen2M2*nM9s;j23^kQDR#&HLZ2};TYPvm` z0Ek})2evcBsaqb8Kp2q6uJc9yDngDQ48qhc$=47(6aY#9cP$@SByIj{Ru`;_D+57a zIXUB$tB{+HV4{>vLY*NH`Lp5PhO|ZbzV&xVW3=dwI>KW%>hHIN5hi+qCd70R91@#7_JEx}gh zyaHbkez7ah3Gtt~wa?9{m2W~~Rg`Wm2fvqSQ4iFbk$ssgq5^f5i65VOXnF>*S&Is= zfdAOuT~`X!vdKwu;1JThM@~u7fDT-sMnld906Y)frp8h=AX-g;OW=4p_@oK{RD*`i zKBWvaP=Y-gN)i{a)PQXeIat}6z6n0BO==U|@h=5^_E0Xp0;?VXC=GJoP81RU?(vn@ z*u>vlzJ&{f^nvrVq_Y|@iAHGw;00e6T7%R9r0FJ5eUt*n{Y{mGr`q?6pufww08N2? zum0OXrvcZq$r~Q5Hjo2n*_6?r&g>!S%SxcxY(2wOCtqF9Jj;2%j~W-OEu^K(h0~ z6?fv74ROiyN<|0MRIw8Rp+StX3r(Y_8SjD#aHbp>J9U}bNn83xAK1;HK9G~AFt%&^acDWj z`sj*Q&|buxV_6s`Dztw*TLy>jxvcPF10Mp(DfGmvKoCtJ(9`*cPWAeh(v2QnFrJ8L zPd?mHxGVkdct3^CgE@-4MH?=+KwN5?L6#2Y>yN0mFR$szBM4uX5THiOnFQgp)Hp@+ z6GOCx9P{hETl7L}LyBqlCJqo7`O7rADB8=bH7)F9>(BZ7a}jz3)#$`99%AwUE%F2j72GFy3H%hC_w&HsvF14|MbTB z#!)orCA}83adMW%>h*lY=@mq?PD-&qICO8 zkGE$&uJrtPcl)ZPKk5wJ3O#rOW%~rOMgtvM-L#2HuoPh_5)S~Lec9rH!Yp1K2LQPR zE!nBhUfUf>z2MkrxjwX6fy_%Wk>=goD?TkICG|DkCppIiMbscVY%RS|T%aI%?>B#x zXdJYG7lSO{qbm_~k(jV|WMh8W$z}!=@FuTkK4JGMY;jo*Z(vl;ESQ>k&6nR$*%Lu# z8116qRcNe(YO}0$6sFH97e-zlJAk5jK|WJ+`{uJ3(*PR$ikLNHckL+%4fZ+>+zq+d zHc$o_vQAmL8q&R7lF*9ysx`OYXl5p~j#&XdFCC?KQ-}2$EyGsEds#Eb=MITq7_xlL zg_;jLt1|c2SV2qPw3Ul=L7(To=0&~%QElahGONmDna1=*K_OIcmND$;e{B6-0$?mu zb9uR~FWb_ad@i&tG6{&%6d;^;$6=3#{rD!cuHanom~~yXw6M16$xTTv>mL()g7fg^ zoUsBSIUH=;FV26w>#3DrUc#RRyR&*TT9 zh&0GqD>_6`_0SKqdu>cp@u4a!+mlK6S|aAn9%Me%JEHV;T+o)cFDjnuUXFX7sCz*- z#vQjD8|?SdjF89yNfX_A< z%dBrW-`HBx-Vn%nwCl>@St;i2vb)Q2Qpwio^_KmkKE67U%#Z8kvMhnEUMiyN6|zC7 zN$Q-_EjFlPyAZ1e98|04N77G*4F?8oCLJhDUEy`q;WcEPDkllc5k!`Q&%(VSE$46e zqy*~aJVr3|YJg|PV@>|^lxz?IKkgc*^%{CWvMgs`7n2b42MC6~ZlHcSD6$5EC2CRP zPcdSX`cc$382jd|JaE_P@f^!Fde?Eh9d*pna~%LR%0b~U+2fNeXuRli0k4QVcrsEp zH2QK<4kDKgLASm!tr*_3j7o-&e z8KpvGJhryv*93c8THRV-+GDs2*5m(tz%89*KtBK!AU`YSn@^xgatQdMB0Xof2W=mQ zz}pZxL?ewcaff#9r-|^8ya=t%UMQ}?m;qlcP1A3(tOgBNfA(LlJq z4QOCW#F+yDzVSPrR^_Gy5 z+wxD^TfJP-pPE~CX-aKUa@IIaL|PA5FDfCd?=3X?Xj&EW9bxA zAYjx-?{Ck%v%@XM5ndN9Wtv#vdMJEp1+w`5Xmw~reBnl%VRZhawBDQs*e2ST{zUY6 zrggP)yfO1`<`4Lrs}kyaz2bDm_1PKMeq2a>UK=!ePUcJ3gmx*NYzR?fF7$S%1Z@WO zBR~3wckO{5&e^7SOmuo=J#?=kvFt{`kcp-y|6+uK^<^>#sGLvhw3-z;?;FAAB_0#4 z^=f3Ug3rB6^hGq^x+3M4=;bfP`R272ow&%qrr8jMrqMUL*s?ll^|sZFmmBQ!P72G? z3O16ustFE0iyjq~&IdQ_O_fQlVx=MV9Z*f79+F2u6I-8-0H^O*&IS3GL_BiJiW zG`Bm6)^smh=!8!>dBOc)TB&Hxs?~WoCqN>Mtv&0^G(g9i&L|8m0J}RZwx)y>&ucQB z^`_^3nlobImr&@hfBw;7J%?%A2cl*H7g863u{4b-O$h8_?FUHj8s-O}kimNq*=LrE z2w^sOy5*`&mV3R8CxAxFxp`&)5TYpeT7N);k+hM_k{JjZEqCF@LJ=wluJH?1hc+$~ zV|Gy)1<;($O6H*6Pu10>EbPZ}0J{eBkButkM2+Jv$XwZu9c*7JgpkR(`#2)%8{|%! zo>>%zrv;3-B5L$Jht2x)g#eWeQgKuSoMU%}?7Tn&&4KyRmkOgOUh{I##qE#Y?r!7h zZM(u7KDE3$4!fEMr23S)l}lN|YGDVOAaCyXVZ3VjZ8w`iq&&X>pkuaecClvXz>o5E zB|;Oh1D)n!R{*#d3TbPGZ~GllId}`XG?roKc`5%K|I$XG>7N0mX=-OQ4f3CqCh{8luVHMM^gbb~t;Y1k z57}8Rq2XEn&aj*Y!VGg6rV(nbeTEztI?J{<-pUiFC|N$Sm$#_EkR(i%N8U48ww{7~ zBD64#nYY>myyrxJF+>V+!z4g{!v$soG+bts@vQ=Lk5TLm0v^63Bh)PD%{Q*6=kD#u zEy4kz^urh5Kx@=QiAo&AL}bRx_#6p|hTAq5XFQ#|H%eAID%0(wooopp6*?>~Q^^43 z$*~{Ud4h{~nxPq!sOV|YK-7K;*Xe@6g<)PmalAB^*OUGY3kpODfMvx*VTyxntQKJL zqz8s9_?~sv*X^KiLWiE+@I{L{Isq+2WpGcH2~n&j($xw?hQp!~uG4_RwJn-h0?vNw zCgw9tN}`OeF>*i0_w=RZ9(FUSNt6}~j5FzIxQDxsmNRc!3Mi1f=onDA(48+$0YX#~dkuseh0nZ6bmkCbdgSt9E5KhX@dBU+RRDqs7zwXx!olx7k~ugx ziYzw{chi^;M05ehLnuR*c%M@5!y}bVlmyOKTRB1Bk7Qfs<)*X52nbpnDD`FDK$Fl8 zV;3R!YMBC?7`ImvmFYhT*PIk zlg1OZpxv}Ia}@QerC!KITk{c74p<>mycJ)W0>I(s?83#_@(P5P3d!PK4iO-IIA@o& z9JH>Rk6|F@T=LzQHn^?CBSC?%fE&0D(qi?|m5zD)SlQjRz^&|=p-hC69R01bh@uwl z3uz64yPRQ6)L93l4}|m@#B`@2eT98`!prN`v*QOPi7FAP{N#2oq=gKbVvU$2Kr8sz z<;?P?yG?rmlNmIS$nIUq#xI4W8)@N7HFr=1d{lW*UQku2XWr_z-qH?K1_R@T&Pigy z-A`MVCL4}EXD}oz4RRr!Df@SkG_0K)5Y58q-sFw$qk}n+ z)MxVI>Jpp{>wdwS!zTSZP%axw&Nc5@mS=rFU$y^(j_^S*}mH~h|dz-L! zRy)C%*XEcF5zpw&fvfFmfGo%nH09bVMPM|V%me`kR2nYVW>6H!#mtR4V1ttg;C!)A z{3tf(-Kcm0J)7Bt-ZGiJh>xTX!dTtH3*^!3HP@1e9G~Hxi!sNg|?y8MY_5e+`mtG}gRHbP=UZ{;cHXrlY zZBWld?SVScVT4(H_(cSt&XQ)qPoO6QGn0(X9a{DNk}R2YBG6?S^piny*})MIev-NG z!{IENq(YRN#gk@np6T=?(jX4ehMW5<6>UA8n=BIr&qHmcSA`q7#w25dQtxk#YtBst zt}zg;Wk(2bnwp&uPb(bz5zenWRKNqA)tJgG-*_1K*8skBshevbDofPAIH4g=`lLm& zK}1i9Jt*0nvv{Q`Yh4jAos+>v9!cmdotB0$Co%^mehw}F7j1X`4`ty0`~SLT-)HPQ zW672@mNZ&4b}BVQh%ypQh#@hgTr-%lW{5&)ES2`rM*CQj(lSbgwviSsYFd=mIalv< zzTb2Ep3nD}b8ffu2UKouVJ>rjK3>&a)QW_5BW^(s{2ni(C~c`>#K?vt_TZBXm>z*R zja7gRB%W))+LR$;ID5@&5t9buqB?QTjCcdHOouDl!8Y2ziQtL@Vt9moGg$HV0{Bk1 zUTuV{K185LHwS3*mB^5(5kLX*2hfrt(!eC-H-I@~fDMQ2G3|5IUq|y(Sf#bI5Y24% zAkgRECCalECoqxK%;J^h%%{PnCQ|?H9C6|Z>ZDWo3Jx~c7F!>B_*-yc8at$H4aJy& z+5bl5M@PK+gvLyw>Kzj@XT*FZvl=q$sDu3sHf4*zvXGoff9c0IJ-ci$b3o)BFrvq( zyV_TfAZ-FM)l!I#P#}|e1)r`;9XA0KK*Z5UCC-R>5g0--bgvJcI@#*ZL_`OO!<0a! zlSXPlaw@OjQS{oY7iR64}^9K4!f>GVFG**=Z7}%#SHTqIYnW=OB5!58NfrIG3D4NKzAIR+xC!zv&3 z>X+31JZK(seUHF6U!JO7>y9srfX~p~!^VQfBe?1=ft_lvChJ7y3ZikX&?EqV(rd1+ zTNkVL@yAn z;IeV{99A=A%|%cXB_!Em{BybyZlzI#CJwiYJRo=;Po=_`SYm#L3y~F&@qj>u(?W7m z#!q+WNp7j9k`aprsU`9!c>@q$0qOe)vdpx=X=n$v0!Ifmxd<~k1KUx834jcgfEEPr zQGm;<1wZ5f#daoNEmzzENW9JDfP7;PRXgA*p8Z6dC9oL~STuJSE<8!*Qs;DNq8;xx zw?iuj3iX&Dr=R8=|M+v!kI!a{K8?S%+B!P!Yw?Lf-6pMVyB+9yI;B*4b$tG(hPmGx zt|`Xd`?TOk?-*9;vP$vJF_Ti+r83;*@wW~1CBHiN{CZlch2C?n$q$&Q*j8rbN`*m^Svi=IKyVQ^9TiZ}`9_O`xJ%jz|d!yzhyY1pW9TQ5i zS}V`MSc2a4*sS8RD5(fzbMrs~&UDA!`2%^sb_jP}(>Kw$=Dpnh)A09?-#X&0H~h^j z+UYC2iJnBowS&8iw$1-hrMN0B!F@lsPGm(}(xRvJrt1f}R;?PKRx7iA?4f)+_2c1L z&(}Z^N&D}gUeyV(kf8H7cv5x38%ffz;K3teb5DwqD|dvg;f=|+V$fIt&9R=P`b=6f zi?%_8(Q-|hnGYgwN;KTke_xC3DZuE=r)|%6es!bVNa)6HSG@=nhN$7!{IB&aFe{o@ z{QsKhcV356CPeWJ3H{0Otb=~r7iMyX1ti=Dog{Vm@rOMjIxS&G0qTV$5%bJ!e}6#{ z1^V(C{pexph=8*Ann_u$bd!v*kh?;EkITkk>EKR}s0gP=gf6`Wl1_snQ= z;D0yKKdh>Qm%8t-&cAA2<~H^_1XD<~PzTPcjV*Cch`2O_ZmT7=;~xhe4dw~w_Q~X| zq<0-fdr~*N9eoO6X~uy!I^(S@8nlCk@$t$maODxBux7>3&y?@m z)+^h89L;`Jh1MyejJ~tq8}sdeoKt7GY)&`YD3{R#$DXfaU&NT%QKAtw+_=R5|nS3&N?~N663#P>(pw+ipAi@7R! z2Va*i_(EBz>LxB(^y#Y6zTM{>_uZbJx>on&LapMI-rO(OqVg7h8%?^h{o$!C>cunT zrAs`1y!@DQ;m4aJAH!MyUk6S=h5TIp6wGPm$K_bvL{I7YF|I=DGjT#tu4v8g669RV}@B{!CI8Vcp>In{WEx zKkC{tchS)codHjM-~D`LpWF3mB2{q3amWSv`L$wBR?&*>@nvn~!!ZfcOF8c=?p-KV zpJ>nhE>}?+D`M`fOUbO5d^EUk<(12Se$9M&_w3Gv4x)@b-79b&bnPGB)Ls0 z^J)LD1J_Sz9~T*CPL7M|BHQOhR=c^+i)|0IKQD1OJNdkn(Qi9Z=K6p;Q9k7)nAo}K z+vG$=T^PGg!n(YAU1flyc1+N-*tUioy`Zf^mUd`ld{tvH+?9^l-G?Y48J`O=Fj-qg z+oL9R(=1q9uh-L4`p+q}v*x8s>QKR62G!}`-fUcbxZt#*=S_UyYnE8owWKl6?JalX zdOvip@3QsPlaboM=g9*N*1j8-)gNbsNgH=3+LzWem5gpXeD>|zyrY-Z_D5Lc=moQr zRz1yF8+7RD+t{O*b#>o(@!i}AHNSPTs_FST8*Z!)$1Hq$kOacczQ<#D!@7}#(^H#< zDDlTmzqp}u+-qV5H);Em-ASPr7B%f zz3#1XTRI->A!3f0Zuwpz@k-ZwwL}+NKK&STEc!dk(WM~dIx2FVY3c$?ho3dMOIB>C zY57>Tl?~l{Kl6HE>2SBwZoKL4z0#x?(?OfFZf6XIED2?d&EBeHo|Lw_x_Mujf3xO` z<@Cf@RkDG)ZfAYyJWJ6oDX{dp{;laDtHR;q_Ilz{lDf{iO*5el z30rf*g-*yi{dO!!r(*yH3*o4U)8+ygd6t*h^k}@Jce4B5T-w>v8Ym`zK$^gl|re56QgDFBP9!GX(0)=E`eMe!0O| z>tw$2UW@6Xp@KEzLpvw8)ob}H1n5u*h$`|mB+0Ki8nWpGeWA^IL1|uFETUoq3qvpu z#Z3zIasr_Fi}7Or-w^1b^XLG*z2}~Xn-nP>il+SwtlbPm@SsL zU~+!B40hmMLPXhu8obBxrL#A9TGlEV_T*ig@40(gK9+~)AC=(7xEQi=|H8A|gOA;9 zi0{&76$CeU0-bSy-f{tEWe8{6)1=nNrys1yR%_y^m*IGE%eWWiIiS(V_p!M-jPqG8 zh?gVeU+el-E-X@x!4{-n?sO$(&lOhU=bWtxT_GIA0{_go+i=? zxkK0N^K^XMT)x3|LT@84V)S$;ZI3}RXGyi`JAz;Ln@~Azu+g%!16(ZT8HPYk4kA>FE127r-J)F6C?P>k#Qy+_2 zT1^_IDA%g36|Smh6eDeX07s2N;33nC!8m}mb^Sdj#vfZr=*l*P{)nv)Ds+LTQCu9w zRDp6g4hB*WO9*dr!H7AXhrTc1@9y$`kGpSZy}TbZ<;6bbC5t;DnmTbQ^xL2- zMot=S7ds2GmzL`$7ofEO{VIX@Y382Xd+x}E%SSzhMc>R^rJ&%~wcU~FJ8L{JQ#C1i z(BjEBZIB6~yV&I^Ji&#txebQhb!X;#?A`s^Qn($P6p``}X{su^wq&G|?AhgNBSRjD z#IHZ`Hoe(kTL0RTSD&cN&@VOe4ztfKft7^tO=wW=nQMY~f&avMB5bK)>!#ghonQv2Nr+iP> zrGb}pGv1UC)OMFPam`HzS4(=7Fm*w7~Q(R{Lf*0SlV~EUY0H#XbuVa(Rcs8 zOGhKFs1kzrla;{peVhc8rfNrZl*ND{D`OEA1>{7WZVVgy_?{nGYRqSu$%e#dznlT< zB0eMS1?4^?mpMe1kMj%=HNG`4fEMb8ULpUjWxN;TMf;Acyo31K-)vgWhHytS5atTb zVD7C`>O;Ycj~Xv+Lx7O)y!W$Fe-h6J-G0hJqlU=ydAhT7T^(TVcD5lY*7%$~5FdOlYa>!E9H915z{7& z;A}f11oaunAbCJjc)%PE!decNw}rqkZ;OnyNd*|uz`_muf09ZGx+{@Ja&V`%zm{r! z2OB6J-g!#QwN+Fc-2+<5L2EiqIgo?%24Z0fr2=S5iLo43-xc-@Q72GM^md_$@VM&@?o#rVC zCpjp`Yjz80_8hgK4*0BRMqhQvU_z*Qxu zTs*Kz6rVW9*1Br|4S%;scB3QMqNFYG`D0=^KrMpSWVaJu^3bCP03Ixt`_rYkkLUuE~q%kYLOG zg-*J*btzo9c7?ETu5;Eyc$kF!BU@`3P8E-Ww=+2L_|(g?6>Is#`un(e1L|cuuysHA zKo})0o9LyM#hb{nqYBCtZ?}WeYqsSECMW;9p3!>+$&s6+)dp;%HLB>uU0q&Go>*H+ zsEMSMOT;kb>)UNoE+Lse1ZT4dN<4K;g`?^L2iTNnZtHlA*vB>SduP<9Y_d?kG24wW z0ql9gYR;c_hldZFx9!rv z22kSxOoxct4`Fh{FmONnV*)9f1)g6`KF%S{C<$q7@;4rFt_rGa1so)#E1aeohE@^Y33mP__&h%R7NVndKgQw zgDUb^Jz-W|RpeKyHH|nZLF&IjcF>4NR1`T4BUh3y(on|m)Ixz7lpceR_*%0o$ZT96 z9K5Cx>i;v%fGhnT(hYE^dRRTyVPE6BC~zB|<12W9gUL6QfqlxW&awt|7(kJNB$mcB zv}~Dx%jU|%{Q%axzRp3_e3*qgfu}aIgAgh_Ec8?4=)=64B95nzf|NQ7l_epi=J1R& zh&f@ApM7Y}FNhCe>8+x=%~0a$G5{ngaUQP6?Ca-$ut zT#ntRsS$ZJ>%q?PsBVSi5gAicicVi~&S=sZpWk z;rY`25j^s~bJ*oP;#j;Vg45LxV3=^+T`8{cUIPo***7-oNDfb5U(h;(n~@NGSjbj* ziJ)4l!&-Wdby1|iM@ugrP*BT(*1t$`BZGRJ_CK)qm>zIkE-?F@)<^>^A0W#GI0DkQKtyC%fdNW^BbxvGU3l;6n!CJG$>cFI8CFtDr>>|&7IImb$^2(a#M=Aqz)!W|`& z#sT0`yNZu0hK;v$r#=S}1QExj_!tIuGY=`#BaF*$`75x43{uISQw9w14)dsaF&M=r zK2Q+y71&L5z5xIfs>oe5jECYRY`irBS=UO)JLwh9rNry?x8-bnG5si73U43CXOw6+ z*p@@Y`E|f%69e6bT1D8_W;xb ztj*D?OZ6hnj9=%+YiK4cjFvMwzMQ8a_n`4`R{|;0Oq}lh-eSyzw+u=kdDo)>3Kk^IuKZpgVu)MrD9Mx%0L2y z`@|sJZv$2@qO|>V9W13b(s6nNDfgO@w`gR;`3NDCdZTY+5k!5cz)clvf@&UmOh(=y z-{_?v^-CW3(a6`N$XG1CFKc{7VRLnEi~dc(W` zc;mqXT3MBWCOrIMX&MMiZ-8iK+{5Mcn0{{urGOuuB9n={A#B%$pf);jQU?fT!}

QHF{TraaWMCPe<~VY|lR6FW5x6o?=Iq`|UV4hCWT zblAv~it46eI}r7bb>Z*9K}IrEi?pyNLA z$PXdvNxA3jHslj|)QvCHjf#oWTE$E^ZvPR2I_=g&C*cV_+h)Z4}LoK zX7&vL0YyIH;rGy^_bPB+EYOQVmC*qU&aVe&*XAA|{FN6Rwc20-&*(V3JNbcF^_~mc z2LN5=Q@sxe0}njN0~XN8Patd*cUI@B>}@*e2^Fy}W7eDnD3$UM_VaNw)Eq)l2vaOb00$;%A+X6ITPTzSsM$h1P(g2{bFp%KmdawuC*YXUdhfm+tmG{+}p%}NC z%XWmM2U_pV_#9~4TAjYs{>bUiOC3%Pr!OVIv|9(00g1KI3J$gs*dV}?(SC1z_AwpT z+ZdvSnw*Lp?b!Ewa{7O4G-i!&5*4}4x!}83-IBAk>N;tGWMyl)X^Z^)EcnQnsR!ym zsyo))AzT|d8}N90&I&qfJGr4@vE$p%X5eGUzsBt#!rL^$#~#@f;w9}ciPrv)c3N>pAyus7v*+3P(T@DhZQxnca!>BA_4ji!+^@7q@B>d!PA z@%y`18*NrP9mpqq=o3E(4box}Z^#Wj+mkYIbgk?g*ZqvsMt&SLTZIS+jIcbp=>WU9 zs9j{jABo*)%s>7ku_Pzd=}7az4epV1nCCMFPm^-(Y-FJq+NyEk=VK2xq)XEW=Vp&+ zn?v4j$E^S&is{pk-S6aAwQ|{v_<%a<#+whI`GXtlebQOi_48IT8!6^Z74!-o*ZsAS%p$~ny%oT+<1@x{keXc zQhyH9)(lo|)I88|Qv4vs!2X!lHd$j$FpmGX za^r!!V+1;dZyGF3CZM2MO)h>{dl{iy-uUlL>dhoM<_2mjcjk4xd{xjh_<^A+UfV5CyV^T9i#cDFdTevtD3fF@PmVysC z|APB0g5g(s(umOG11FXc9bpy>$ccmPXaZ9%pk?yGqQIH^ zlWrR~u?%k2nu-CtbZ`5+KXOmbV)h5`+|m|jGR41UpM70BRw%Z+kzp;p+RO$geU>~A z6{Nd*TUpiyG$B;Nn6fzSiY_?Q;6(5~J?LyBLAUS`oz_!w+I|>xFYPD}f2g@@Sw+D0R0mfyp^ZL=+6VEn)nfvudN|?NPiv4iiJF6<(#}2&<&W z=fL}WLjze*IQ^C@%yArU_5k5GCkV&W@5A%83P1jq5K1sasT%TkP)bQpX0Cccx^>Mu zQ(Ov1&m?82JSQ5d@pOIn%0K~Hm(jh++?@zs>_2+!Pj5PtyWg+cn&O%P=nhFiV!wGQ zhWWO9X_e6v2Eq=l0L(}=xRnRK&5x5VwZI<2AlHA!gZiGP^P7bpwC-K%^LB4g{R5b0 zA39X*cohvwy%(EmKnn^t*f%PzEgTN5@P-NyAn@?~Sh^Luk7Bx9R`z589bi;@8#PFU z8nhw*wqgK2#qrd#OnAc8=}z_mubR0du>_CW2UWS!9;dy<_Lm2QCeKbmsC6>p=ej$t zTecTf2oFL=B&42U5rn^c5BfRP?}kx<^LWiIUVNnfcBYW*8qC7^&=L4PKMPU5C4$8k zJ`ZFWUUX`-NtDv=*!BrL0|Xi#6N~4s#{oZOUV#2&p%V%y)08|zr*IY-DVquH<8<5^ z1hAF9zG!@KpVb)0po?LF0oeF-t50Av&VM#$NbF&yZlP`ABCv`tR9q;+Xnk(u&jSWl z7}-$3#2IJPOTXh8;>eqEnN_Bn?ovXGPnlHVpxG}zjqE$;WwvpqhFL%EjN$cEI!b-r zCym|r_34J#Fo(6zx3ydQg1Q4LUnk=!V$7tF6dx7?+-B_DkkO;vHskO3%1P^rZ59gT zqC7liI}mWYs2fL%_K#N17U3YuH(nv2ubjsC@2#;RnVltew*bW!?aw~@5NmoKo@MXGA#Cr@sg5)} zUb3@QSBvGeDP6PaYboeD)CBnOMi;-I4XA%*`Y!I`jaL^$ZOe`H#1=_QT7z8oKW*t@ zrgYDnZ#m8JBSJ`H_HvQydeetp0-raPcUz%Y(@XWke%i9x35Dh6L$)KHM(Lybn?1Y8 z=EjSh(mlSqw>4&W;1b7rfJ;BU08Nc2pU!(8bZpc%9fF@RCR?D*E4;u^8P?+T^QA@0 zGmTeqNJG_{Q1g%=&BlAsa%8ADch&8)@Wa7Ex8Mt7{i}ogpuL;)8O1sLOUXrCYqSHT zfg#-qUE9N4bQ`iS+*g9Q4EY$JP@_{fIu7 z^Ehc>cewUbWjS#l=NZDdS$9yS7GiU+lsrK({f6H3BHcKC2p_sIhE@R@M;9a7>7e@v z9X*I^%Ucc=Y|_7teZ9&|4RHIMT_H@`I7kKtf;G6GN{O^MV3DVZ&eA2@{Z2Q-FVT_K z0AwkQZqYI*zy?shE(7P@0R9A^QUg$AhJeIc{Q0C2JrV*H&cl{2q?;`AV-}^Lief8X zd{>Z8*J5zR{=AYDMg3h47}t(5#sRRH-ptF7UN4AEgckEK>_Gq{LrDw$~O_|=9h zz%8w;@ZVJWHE?k;uMkG%AXAVTE8rH1iY$&X8JcKG;Tcvz9jtuqIAL(Ll-4WeN1G==I!&*aOH94OsTXu zIY7j%!kH^XrLl`Dq5X;Nn7GmTQ-(g5ZJA@)#~ksHiLH3hPCFz*7Z^Zo3n&C;AfUEj zdsm^(WNCy1%VI6WDtqpP0CviPIASTONR+NG+;|fkf`bJV6g(-uy{acq;T=H-Vt(kj z3=3BO)M@dY?Xv;U9l#_Aj%p~8k!%xsaB&_3#b*?7(+vnxK@NL?`oKa>LRVoLFF(Sn zsg5If)_s1F24pA0deTU;Q%3IEq8vH$^cNyuaqvYkQj2t~l4;#?SnD>-kY#(T*5{`M z829&H+*pV8&Nphp>1=-sm~<89upG*`XP7d4cR<(L=feF&poIvuwTpvibn+IWTnDTz z6{5ru0$N|l<%$(i@Wf3JEh!wLmnTGN-m|^3Q7N5~P&)$vjiCZ;d-+d#rLi0hn`ha%uB13+&>yykhN+M>uK0FGC%5r;H3Z2htc@Rw?Q0B8f%!ihv28hfO_(+!PY-2gDGMiUl5)JcQ)se4nSl^OYe$QR${!>nS-|B4F zM34&(2cs?M!l)wgDm!un2M(hnEcmF#@qz_`zL{MpbEVMR$#nM!a#0sD_Fuai>7l$( zf4yk!g{pz>gItBc(Lm76LvJfWhL{)G$?qgbC(UFb4XXy8ids1;wZ+#_Oirm4``nk5w@^E