From 0d99ae1fe84f8d191abe5ed1c2f4fdc5a9f9a773 Mon Sep 17 00:00:00 2001 From: xingjl Date: Fri, 14 Mar 2025 10:53:01 +0800 Subject: [PATCH] add --- CHANGELOG.md | 0 CODEOWNERS | 0 CONTRIBUTING.md | 0 Dockerfile.ci.dev | 0 Dockerfile.ci.lts | 0 Dockerfile.linting | 0 GPT_pretraining.sh | 0 LICENSE | 0 Llama_pretraining.sh | 200 ++-- MANIFEST.in | 0 README.md.origin | 0 docs/llama_mistral.md | 0 docs/source/api-guide/context_parallel.rst | 0 docs/source/api-guide/datasets.rst | 0 docs/source/api-guide/dist_checkpointing.rst | 0 .../dist_checkpointing.strategies.rst | 0 docs/source/api-guide/dist_optimizer.md | 0 docs/source/api-guide/distributed.rst | 0 .../api-guide/encoder_decoder_parallelism.rst | 0 docs/source/api-guide/fusions.rst | 0 docs/source/api-guide/index.rst | 0 docs/source/api-guide/models.bert.rst | 0 docs/source/api-guide/models.gpt.rst | 0 docs/source/api-guide/models.rst | 0 docs/source/api-guide/models.t5.rst | 0 docs/source/api-guide/moe.rst | 0 .../api-guide/num_microbatches_calculator.rst | 0 .../api-guide/optimizer_param_scheduler.rst | 0 docs/source/api-guide/pipeline_parallel.rst | 0 docs/source/api-guide/tensor_parallel.rst | 0 docs/source/api-guide/transformer.rst | 0 .../images/context_parallel/CP_overview.png | Bin .../images/context_parallel/CP_results.png | Bin .../images/distrib_optimizer/data_flow.png | Bin .../distrib_optimizer/sharding_scheme.png | Bin docs/source/images/moe/token_drop.png | Bin docs/source/index.rst | 0 docs/source/user-guide/index.rst | 0 .../detxoify_lm/README.md | 0 .../annotations/filter-selfgeneration.py | 0 .../annotations/perspective_api_annotate.py | 0 .../detxoify_lm/annotations/preprocess.sh | 0 .../detxoify_lm/finetune_gpt.py | 0 .../finetune_gpt_distributed-1.3b.sh | 0 .../detxoify_lm/generate-1.3b.sh | 0 .../detxoify_lm/generate_samples_gpt.py | 0 .../detxoify_lm/perspective_api.py | 0 .../selfgenerate-1.3b-unconditional.sh | 0 .../academic_paper_scripts/msdp/README.md | 0 .../msdp/data_processing.sh | 0 .../msdp/eval_knwl_generation.sh | 0 .../msdp/eval_resp_generation.sh | 0 .../msdp/prep_resp_gen.sh | 0 .../msdp/prompt_knwl_gen.sh | 0 .../msdp/prompt_resp_gen.sh | 0 .../academic_paper_scripts/sc21/CONFIG.sh | 0 .../academic_paper_scripts/sc21/README.md | 0 .../academic_paper_scripts/sc21/SBATCH.sh | 0 examples/academic_paper_scripts/sc21/SRUN.sh | 0 .../sc21/run_figure_11.sh | 0 .../sc21/run_figure_12.sh | 0 .../sc21/run_figure_13.sh | 0 .../sc21/run_figure_14.sh | 0 .../sc21/run_figure_15.sh | 0 .../sc21/run_figure_16.sh | 0 .../sc21/run_figure_17.sh | 0 .../sc21/run_figure_18.sh | 0 .../sc21/run_table_1.sh | 0 examples/bert/README.md | 0 examples/bert/train_bert_340m_distributed.sh | 0 examples/export/README.md | 0 .../pretrain_gpt_modelopt.py | 0 .../export/ptq_and_trtllm_export/README.md | 0 .../ptq_trtllm_llama2_7b.sh | 0 .../ptq_trtllm_llama3_1_8b.sh | 0 .../ptq_trtllm_llama3_8b.sh | 0 .../ptq_trtllm_minitron_8b.sh | 0 .../ptq_trtllm_mistral_12b.sh | 0 .../ptq_trtllm_mixtral_8x7b.sh | 0 .../text_generation_ptq.py | 0 .../trtllm_text_generation.py | 0 examples/export/trtllm_export/README.md | 0 .../gpt_distributed_gpu_export.py | 0 .../gpt_single_device_cpu_export.py | 0 examples/gpt3/README.md | 0 examples/gpt3/gpt_config.yaml | 1 + examples/gpt3/train_gpt3_175b_distributed.sh | 0 examples/inference/README.md | 105 ++- ...ch_inference.py => gpt_batch_inference.py} | 10 +- .../llama_mistral/huggingface_reference.py | 0 .../run_text_generation_llama3.1.sh | 0 .../run_text_generation_llama3.sh | 0 .../run_text_generation_mistral.sh | 0 .../run_text_generation_server_345M.sh | 0 ...eneration_server_345M_8_tensor_parallel.sh | 0 .../inference/t5/simple_t5_batch_inference.py | 6 +- examples/mamba/.gitignore | 0 examples/mamba/Dockerfile | 0 examples/mamba/README.md | 0 examples/mamba/run_text_gen_server_8b.sh | 0 examples/mamba/run_text_gen_server_8b_gpt3.sh | 0 examples/mamba/train.sh | 0 examples/mixtral/README.md | 0 .../mixtral/train_mixtral_8x7b_distributed.sh | 0 examples/multimodal/Dockerfile | 0 examples/multimodal/README.md | 4 +- .../multimodal/assets/pretrain_curves.png | Bin .../combine_lm_vision_checkpoints.sh | 0 examples/multimodal/combine_state_dicts.py | 0 examples/multimodal/config.py | 50 +- .../convert_llava_pretrain_to_wds.py | 0 examples/multimodal/dataloader_provider.py | 0 examples/multimodal/dataset_helpers.py | 203 ++-- .../{ => evaluation}/evaluate_ai2d.py | 22 +- .../{ => evaluation}/evaluate_chartqa.py | 13 +- .../{ => evaluation}/evaluate_coco.py | 18 +- .../{ => evaluation}/evaluate_mathvista.py | 12 +- .../{ => evaluation}/evaluate_mmmu.py | 10 + .../{ => evaluation}/evaluate_ocrbench.py | 12 +- .../{ => evaluation}/evaluate_textvqa.py | 25 +- .../{ => evaluation}/evaluate_vqav2.py | 16 +- .../{ => evaluation}/evaluation_datasets.py | 84 +- examples/multimodal/image_processing.py | 0 examples/multimodal/layer_specs.py | 0 examples/multimodal/manual_prompts.json | 0 examples/multimodal/model.py | 14 + .../model_converter/clip_converter.py | 0 .../model_converter/internvit_converter.py | 0 .../model_converter/siglip_converter.py | 0 .../model_converter/vision_model_tester.py | 0 examples/multimodal/multimodal_args.py | 6 +- examples/multimodal/nvlm/README.md | 11 +- examples/multimodal/nvlm/internvit.py | 0 examples/multimodal/nvlm/nvlm_prompts.json | 0 .../nvlm/pp_checkpoint_converter.py | 0 examples/multimodal/nvlm/pretrain_blend.yaml | 0 .../nvlm/pretrain_qwen20_72b_internvit_6b.sh | 2 +- .../nvlm/pretrain_yi_34b_internvit_6b.sh | 2 +- ...text_generation_qwen20_72b_internvit_6b.sh | 2 +- .../run_text_generation_qwen25_7b_siglip.sh | 111 +++ ...run_text_generation_yi_34b_internvit_6b.sh | 4 +- examples/multimodal/nvlm/sft_34b_internvit.sh | 2 +- examples/multimodal/nvlm/sft_blend.yaml | 0 .../nvlm/sft_qwen20_72b_internvit_6b.sh | 2 +- examples/multimodal/pretrain_dataset.yaml | 0 examples/multimodal/pretrain_mistral_clip.sh | 7 +- examples/multimodal/run_text_generation.py | 28 +- examples/multimodal/sft_dataset.yaml | 0 examples/multimodal/sft_mistral_clip.sh | 7 +- .../text_generation_mistral_clip.sh | 17 +- examples/multimodal/train.py | 4 +- examples/retro/README.md | 0 examples/retro/preprocess_data.sh | 0 examples/retro/train_retro_2b_distributed.sh | 0 examples/run_simple_mcore_train_loop.py | 0 examples/t5/README.md | 0 examples/t5/t5_mcore_train_curve.png | Bin examples/t5/train_t5_220m_distributed.sh | 0 hostfile_gptmoe | 16 + hostfile_mixtral8x7B | 2 + images/model_table.png | Bin images/strong_scaling.png | Bin images/weak_scaling.png | Bin kill.sh | 3 + megatron/core/QuickStart.md | 0 megatron/core/README.md | 0 megatron/core/README_STRAGGLER.md | 0 megatron/core/__init__.py | 0 megatron/core/config_logger.py | 0 megatron/core/datasets/Makefile | 0 megatron/core/datasets/__init__.py | 0 megatron/core/datasets/bert_dataset.py | 0 megatron/core/datasets/blended_dataset.py | 0 .../blended_megatron_dataset_builder.py | 0 .../blended_megatron_dataset_config.py | 0 megatron/core/datasets/gpt_dataset.py | 0 megatron/core/datasets/helpers.cpp | 0 megatron/core/datasets/helpers.py | 0 ...rs_cpp.cpython-310-x86_64-linux-gnu.so_bak | Bin 0 -> 247072 bytes megatron/core/datasets/indexed_dataset.py | 0 megatron/core/datasets/masked_dataset.py | 0 megatron/core/datasets/megatron_dataset.py | 0 megatron/core/datasets/megatron_tokenizer.py | 0 megatron/core/datasets/multimodal_dataset.py | 0 megatron/core/datasets/readme.md | 0 megatron/core/datasets/retro/__init__.py | 0 .../core/datasets/retro/config/__init__.py | 0 .../datasets/retro/config/bert_embedders.py | 0 megatron/core/datasets/retro/config/config.py | 0 .../retro/config/gpt_chunk_datasets.py | 0 .../core/datasets/retro/config/tokenizers.py | 0 megatron/core/datasets/retro/db/__init__.py | 0 megatron/core/datasets/retro/db/build.py | 0 megatron/core/datasets/retro/db/dataset.py | 0 megatron/core/datasets/retro/db/utils.py | 0 megatron/core/datasets/retro/external_libs.py | 0 .../core/datasets/retro/index/__init__.py | 0 megatron/core/datasets/retro/index/build.py | 0 megatron/core/datasets/retro/index/factory.py | 0 megatron/core/datasets/retro/index/index.py | 0 .../datasets/retro/index/indexes/__init__.py | 0 .../retro/index/indexes/faiss_base.py | 0 .../retro/index/indexes/faiss_par_add.py | 0 megatron/core/datasets/retro/index/utils.py | 0 .../core/datasets/retro/index/validate.py | 0 .../core/datasets/retro/query/__init__.py | 0 .../datasets/retro/query/gpt_chunk_dataset.py | 0 .../retro/query/multi_split_gpt_dataset.py | 0 megatron/core/datasets/retro/query/query.py | 0 .../datasets/retro/query/retro_dataset.py | 0 megatron/core/datasets/retro/query/utils.py | 0 megatron/core/datasets/retro/utils.py | 0 megatron/core/datasets/t5_dataset.py | 0 megatron/core/datasets/utils.py | 0 megatron/core/datasets/utils_s3.py | 0 megatron/core/dist_checkpointing/__init__.py | 0 megatron/core/dist_checkpointing/core.py | 0 .../core/dist_checkpointing/dict_utils.py | 0 .../core/dist_checkpointing/exchange_utils.py | 0 megatron/core/dist_checkpointing/mapping.py | 3 +- megatron/core/dist_checkpointing/optimizer.py | 0 .../core/dist_checkpointing/serialization.py | 2 - .../state_dict_transformation.py | 0 .../dist_checkpointing/strategies/__init__.py | 0 .../strategies/async_utils.py | 0 .../dist_checkpointing/strategies/base.py | 0 .../dist_checkpointing/strategies/common.py | 0 .../strategies/filesystem_async.py | 0 .../strategies/fully_parallel.py | 0 .../strategies/resharding.py | 0 .../strategies/state_dict_saver.py | 0 .../strategies/tensorstore.py | 0 .../dist_checkpointing/strategies/torch.py | 0 .../strategies/two_stage.py | 0 .../dist_checkpointing/strategies/zarr.py | 0 megatron/core/dist_checkpointing/utils.py | 0 .../core/dist_checkpointing/validation.py | 27 +- megatron/core/distributed/README.md | 0 megatron/core/distributed/__init__.py | 0 .../core/distributed/data_parallel_base.py | 0 .../distributed/distributed_data_parallel.py | 33 +- .../distributed_data_parallel_config.py | 0 .../core/distributed/finalize_model_grads.py | 0 .../core/distributed/param_and_grad_buffer.py | 34 +- .../torch_fully_sharded_data_parallel.py | 0 megatron/core/enums.py | 0 megatron/core/export/__init__.py | 0 megatron/core/export/data_type.py | 0 megatron/core/export/export_config.py | 0 megatron/core/export/model_type.py | 0 megatron/core/export/trtllm/__init__.py | 0 .../export/trtllm/engine_builder/__init__.py | 0 .../engine_builder/trtllm_engine_builder.py | 0 .../trtllm/model_to_trllm_mapping/__init__.py | 0 .../default_conversion_dict.py | 0 .../core/export/trtllm/trt_model_config.py | 0 megatron/core/export/trtllm/trt_model_type.py | 0 megatron/core/export/trtllm/trtllm_helper.py | 0 megatron/core/export/trtllm/trtllm_layers.py | 0 .../trtllm_weights_converter/__init__.py | 0 ...tributed_trtllm_model_weights_converter.py | 0 ...e_device_trtllm_model_weights_converter.py | 0 megatron/core/extensions/__init__.py | 0 .../core/extensions/transformer_engine.py | 49 +- megatron/core/fusions/__init__.py | 0 megatron/core/fusions/fused_bias_dropout.py | 0 megatron/core/fusions/fused_bias_geglu.py | 0 megatron/core/fusions/fused_bias_gelu.py | 0 megatron/core/fusions/fused_bias_swiglu.py | 0 megatron/core/fusions/fused_cross_entropy.py | 0 megatron/core/fusions/fused_layer_norm.py | 0 megatron/core/fusions/fused_softmax.py | 0 megatron/core/inference/__init__.py | 0 .../core/inference/ammo_support/__init__.py | 0 .../inference/ammo_support/gpt/model_specs.py | 0 .../ammo_support/gpt/state_dict_hooks.py | 0 .../core/inference/common_inference_params.py | 33 +- .../core/inference/communication_utils.py | 0 megatron/core/inference/engines/__init__.py | 0 .../core/inference/engines/abstract_engine.py | 0 .../core/inference/engines/mcore_engine.py | 23 +- megatron/core/inference/inference_request.py | 4 +- .../model_inference_wrappers/__init__.py | 0 .../abstract_model_inference_wrapper.py | 0 .../model_inference_wrappers/gpt/__init__.py | 0 .../gpt/gpt_inference_wrapper.py | 0 .../inference_wrapper_config.py | 0 .../model_inference_wrappers/t5/__init__.py | 0 .../t5/t5_inference_wrapper.py | 0 .../inference/modelopt_support/__init__.py | 0 .../modelopt_support/gpt/__init__.py | 0 .../modelopt_support/gpt/model_specs.py | 0 .../modelopt_support/gpt/state_dict_hooks.py | 0 megatron/core/inference/sampling_params.py | 35 + megatron/core/inference/scheduler.py | 6 +- .../text_generation_controllers/__init__.py | 0 ...oder_decoder_text_generation_controller.py | 8 +- .../simple_text_generation_controller.py | 401 +------- .../text_generation_controller.py | 400 ++++++++ megatron/core/inference/utils.py | 0 megatron/core/inference_params.py | 0 megatron/core/jit.py | 16 +- megatron/core/model_parallel_config.py | 0 megatron/core/models/T5/__init__.py | 0 megatron/core/models/T5/t5_model.py | 0 megatron/core/models/T5/t5_spec.py | 0 megatron/core/models/__init__.py | 0 megatron/core/models/bert/__init__.py | 0 megatron/core/models/bert/bert_layer_specs.py | 72 +- megatron/core/models/bert/bert_lm_head.py | 0 megatron/core/models/bert/bert_model.py | 0 megatron/core/models/bert/pooler.py | 0 megatron/core/models/common/__init__.py | 0 .../core/models/common/embeddings/__init__.py | 0 .../embeddings/language_model_embedding.py | 0 .../models/common/embeddings/rope_utils.py | 29 +- .../common/embeddings/rotary_pos_embedding.py | 0 .../embeddings/yarn_rotary_pos_embedding.py | 0 .../models/common/language_module/__init__.py | 0 .../common/language_module/language_module.py | 0 .../models/common/vision_module/__init__.py | 0 .../common/vision_module/vision_module.py | 0 megatron/core/models/gpt/__init__.py | 0 megatron/core/models/gpt/gpt_layer_specs.py | 121 ++- megatron/core/models/gpt/gpt_model.py | 0 megatron/core/models/gpt/moe_module_specs.py | 81 ++ megatron/core/models/mamba/__init__.py | 0 .../core/models/mamba/mamba_layer_specs.py | 0 megatron/core/models/mamba/mamba_model.py | 0 megatron/core/models/multimodal/__init__.py | 0 .../core/models/multimodal/llava_model.py | 3 +- megatron/core/models/multimodal/llava_spec.py | 0 megatron/core/models/retro/__init__.py | 0 megatron/core/models/retro/base_attention.py | 0 megatron/core/models/retro/config.py | 0 .../core/models/retro/decoder_attention.py | 0 megatron/core/models/retro/decoder_spec.py | 0 .../core/models/retro/encoder_attention.py | 0 megatron/core/models/retro/encoder_spec.py | 0 megatron/core/models/retro/model.py | 0 megatron/core/models/retro/utils.py | 0 megatron/core/models/vision/__init__.py | 0 megatron/core/models/vision/clip_vit_model.py | 0 .../models/vision/multimodal_projector.py | 0 .../core/models/vision/vit_layer_specs.py | 0 megatron/core/num_microbatches_calculator.py | 0 megatron/core/optimizer/__init__.py | 73 +- megatron/core/optimizer/clip_grads.py | 30 +- megatron/core/optimizer/distrib_optimizer.py | 295 ++++-- megatron/core/optimizer/grad_scaler.py | 0 megatron/core/optimizer/optimizer.py | 241 +++-- megatron/core/optimizer/optimizer_config.py | 65 ++ megatron/core/optimizer_param_scheduler.py | 0 megatron/core/package_info.py | 0 megatron/core/packed_seq_params.py | 0 megatron/core/parallel_state.py | 0 megatron/core/pipeline_parallel/__init__.py | 0 .../pipeline_parallel/p2p_communication.py | 0 megatron/core/pipeline_parallel/schedules.py | 10 + megatron/core/requirements.txt | 0 megatron/core/rerun_state_machine.py | 165 ++-- megatron/core/ssm/__init__.py | 0 megatron/core/ssm/mamba_block.py | 0 .../core/ssm/mamba_hybrid_layer_allocation.py | 0 megatron/core/ssm/mamba_layer.py | 0 megatron/core/ssm/mamba_mixer.py | 0 megatron/core/ssm/triton_cache_manager.py | 0 megatron/core/tensor_parallel/__init__.py | 0 .../core/tensor_parallel/cross_entropy.py | 1 + megatron/core/tensor_parallel/data.py | 0 megatron/core/tensor_parallel/layers.py | 2 +- megatron/core/tensor_parallel/mappings.py | 6 +- megatron/core/tensor_parallel/random.py | 0 megatron/core/tensor_parallel/utils.py | 0 megatron/core/timers.py | 0 megatron/core/transformer/__init__.py | 0 megatron/core/transformer/attention.py | 0 megatron/core/transformer/cuda_graphs.py | 882 +++++++++++++----- .../transformer/custom_layers/__init__.py | 0 .../custom_layers/transformer_engine.py | 0 .../core/transformer/dot_product_attention.py | 0 megatron/core/transformer/enums.py | 0 megatron/core/transformer/identity_op.py | 0 megatron/core/transformer/mlp.py | 0 megatron/core/transformer/module.py | 0 megatron/core/transformer/moe/README.md | 5 +- megatron/core/transformer/moe/__init__.py | 0 megatron/core/transformer/moe/experts.py | 0 .../core/transformer/moe/grouped_gemm_util.py | 0 .../moe/legacy_a2a_token_dispatcher.py | 0 megatron/core/transformer/moe/moe_layer.py | 23 +- megatron/core/transformer/moe/moe_utils.py | 122 ++- megatron/core/transformer/moe/router.py | 81 +- .../core/transformer/moe/shared_experts.py | 9 +- .../core/transformer/moe/token_dispatcher.py | 3 + .../core/transformer/moe/upcycling_utils.py | 0 .../transformer/multi_latent_attention.py | 0 megatron/core/transformer/spec_utils.py | 0 megatron/core/transformer/torch_layer_norm.py | 0 megatron/core/transformer/torch_norm.py | 0 .../core/transformer/transformer_block.py | 5 +- .../core/transformer/transformer_config.py | 42 +- .../core/transformer/transformer_layer.py | 2 +- megatron/core/transformer/utils.py | 0 megatron/core/utils.py | 38 + megatron/inference/__init__.py | 0 megatron/inference/algos/__init__.py | 0 megatron/inference/algos/distillation.py | 0 megatron/inference/arguments.py | 0 megatron/inference/checkpointing.py | 0 megatron/inference/docs/distillation.md | 0 megatron/inference/endpoints/common.py | 0 megatron/inference/endpoints/completions.py | 0 megatron/inference/gpt/__init__.py | 0 megatron/inference/gpt/loss_func.py | 0 megatron/inference/gpt/model_provider.py | 0 megatron/inference/static/index.html | 0 .../inference/text_generation/__init__.py | 0 megatron/inference/text_generation/api.py | 0 .../inference/text_generation/beam_utils.py | 0 .../text_generation/communication.py | 0 .../inference/text_generation/forward_step.py | 9 +- .../inference/text_generation/generation.py | 0 .../inference/text_generation/sampling.py | 0 .../inference/text_generation/tokenization.py | 0 megatron/inference/text_generation_server.py | 0 megatron/legacy/data/__init__.py | 0 megatron/legacy/data/autoaugment.py | 0 .../legacy/data/biencoder_dataset_utils.py | 0 megatron/legacy/data/data_samplers.py | 0 megatron/legacy/data/dataset_utils.py | 0 megatron/legacy/data/ict_dataset.py | 0 megatron/legacy/data/image_folder.py | 0 megatron/legacy/data/multimodal_dataset.py | 0 megatron/legacy/data/orqa_wiki_dataset.py | 0 megatron/legacy/data/realm_dataset_utils.py | 0 megatron/legacy/data/realm_index.py | 0 megatron/legacy/data/vit_dataset.py | 0 .../legacy/fp16_deprecated/loss_scaler.py | 0 megatron/legacy/fused_kernels/__init__.py | 0 megatron/legacy/fused_kernels/compat.h | 0 .../legacy/fused_kernels/tests/__init__.py | 0 .../fused_kernels/tests/test_fused_kernels.py | 0 megatron/legacy/fused_kernels/type_shim.h | 0 megatron/legacy/indexer.py | 0 megatron/legacy/model/__init__.py | 0 megatron/legacy/model/bert_model.py | 0 megatron/legacy/model/biencoder_model.py | 0 megatron/legacy/model/classification.py | 0 megatron/legacy/model/enums.py | 0 megatron/legacy/model/fused_bias_gelu.py | 0 megatron/legacy/model/fused_layer_norm.py | 0 megatron/legacy/model/fused_softmax.py | 0 megatron/legacy/model/gpt_model.py | 0 megatron/legacy/model/language_model.py | 4 + megatron/legacy/model/module.py | 0 megatron/legacy/model/multiple_choice.py | 0 megatron/legacy/model/realm_model.py | 0 megatron/legacy/model/rms_norm.py | 0 megatron/legacy/model/t5_model.py | 0 megatron/legacy/model/transformer.py | 37 +- megatron/legacy/model/utils.py | 0 .../legacy/model/vision/classification.py | 0 megatron/legacy/model/vision/dino.py | 0 .../model/vision/esvit_swin_backbone.py | 0 megatron/legacy/model/vision/inpainting.py | 0 megatron/legacy/model/vision/knn_monitor.py | 0 megatron/legacy/model/vision/mit_backbone.py | 0 megatron/legacy/model/vision/swin_backbone.py | 0 megatron/legacy/model/vision/utils.py | 0 megatron/legacy/model/vision/vit_backbone.py | 0 megatron/legacy/mpu/tests/__init__.py | 0 megatron/legacy/mpu/tests/commons.py | 0 .../legacy/mpu/tests/test_cross_entropy.py | 0 megatron/legacy/mpu/tests/test_data.py | 0 megatron/legacy/mpu/tests/test_initialize.py | 0 megatron/legacy/mpu/tests/test_layers.py | 0 megatron/legacy/mpu/tests/test_random.py | 0 megatron/training/__init__.py | 0 megatron/training/activations.py | 0 megatron/training/arguments.py | 49 +- megatron/training/async_utils.py | 0 megatron/training/checkpointing.py | 39 +- megatron/training/dist_signal_handler.py | 0 megatron/training/ft_integration.py | 0 megatron/training/global_vars.py | 0 megatron/training/initialize.py | 0 megatron/training/log_handler.py | 0 megatron/training/one_logger_utils.py | 0 megatron/training/theoretical_memory_usage.py | 0 megatron/training/tokenizer/__init__.py | 0 .../training/tokenizer/bert_tokenization.py | 0 .../training/tokenizer/gpt2_tokenization.py | 0 .../tokenizer/multimodal_tokenizer.py | 0 megatron/training/tokenizer/tokenizer.py | 0 megatron/training/training.py | 189 ++-- megatron/training/utils.py | 99 +- megatron/training/yaml_arguments.py | 0 mypy.ini | 0 pretrain_bert.py | 0 pretrain_gpt.py | 6 +- pretrain_ict.py | 0 pretrain_mamba.py | 0 pretrain_retro.py | 0 pretrain_t5.py | 0 pretrain_vision_classify.py | 0 pretrain_vision_dino.py | 0 pretrain_vision_inpaint.py | 0 pretrain_vlm.py | 14 +- pyproject.toml | 0 pytest.ini | 0 requirements.txt | 1 + .../requirements.txt | 0 .../requirements.txt | 0 run.sh | 16 + run_GPT-MOE.sh | 19 + run_GPT-MOE_1nodes.sh | 16 + run_mixtral8x7B_1nodes.sh | 16 + run_mixtral8x7B_4nodes.sh | 19 + setup.py | 0 tasks/data_utils.py | 0 tasks/ensemble_classifier.py | 0 tasks/eval_utils.py | 0 tasks/finetune_utils.py | 0 tasks/glue/data.py | 0 tasks/glue/finetune.py | 0 tasks/glue/mnli.py | 0 tasks/glue/qqp.py | 0 tasks/main.py | 0 tasks/msdp/README.md | 0 tasks/msdp/evaluate.py | 0 tasks/msdp/main.py | 0 tasks/msdp/metrics.py | 0 tasks/msdp/preprocessing.py | 0 tasks/msdp/prompt.py | 0 tasks/orqa/README.md | 0 tasks/orqa/evaluate_orqa.py | 0 tasks/orqa/evaluate_utils.py | 0 tasks/orqa/supervised/data.py | 0 tasks/orqa/supervised/eval_utils.py | 0 tasks/orqa/supervised/finetune.py | 0 tasks/orqa/unsupervised/nq.py | 0 tasks/orqa/unsupervised/qa_utils.py | 0 tasks/orqa/unsupervised/tokenizers.py | 0 tasks/quantize/calibrate_gpt.py | 0 tasks/race/data.py | 0 tasks/race/finetune.py | 0 tasks/vision/classification/classification.py | 0 tasks/vision/classification/eval_utils.py | 0 tasks/vision/finetune_utils.py | 0 tasks/vision/main.py | 0 tasks/vision/segmentation/cityscapes.py | 0 tasks/vision/segmentation/data.py | 0 .../vision/segmentation/finetune_segformer.py | 0 tasks/vision/segmentation/finetune_setr.py | 0 tasks/vision/segmentation/metrics.py | 0 tasks/vision/segmentation/seg_heads.py | 0 tasks/vision/segmentation/seg_models.py | 0 tasks/vision/segmentation/transforms.py | 0 tasks/vision/segmentation/utils.py | 0 tasks/zeroshot_gpt/datasets.py | 0 tasks/zeroshot_gpt/detokenizer.py | 0 tasks/zeroshot_gpt/evaluate.py | 0 tests/__init__.py | 0 tests/functional_tests/__init__.py | 0 .../python_test_utils/__init__.py | 0 .../python_test_utils/common.py | 0 .../get_test_results_from_tensorboard_logs.py | 0 .../python_test_utils/test_ci_pipeline.py | 0 .../python_test_utils/test_fp8_ci_pipeline.py | 0 .../test_resume_checkpoint_pipeline.py | 0 .../shell_test_utils/_run_training.sh | 0 .../shell_test_utils/run_ci_test.sh | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../bert_release/golden_values_0.9.0.json | 0 .../bert/bert_release/model_config.yaml | 0 .../common/ckpt_converter/__main__.py | 0 .../common/ckpt_converter/model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_0.8.0.json | 0 .../golden_values_0.9.0.json | 0 .../gpt/gpt3_15b_8t_release/model_config.yaml | 0 .../gpt3_15b_8t_release_sm/model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 1 + .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 1 - .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 1 - .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 58 +- .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 1 + .../golden_values_0.9.0.json | 0 .../model_config.yaml | 0 .../golden_values_0.8.0.json | 0 .../golden_values_0.9.0.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_0.9.0.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 1 + .../golden_values_lts.json | 1 + .../model_config.yaml | 57 ++ .../golden_values_dev.json | 1 + .../golden_values_lts.json | 1 + .../model_config.yaml | 58 ++ .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../model_config.yaml | 1 + .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 1 + .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 1 + .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../model_config.yaml | 0 .../golden_values_dev.json | 0 .../golden_values_lts.json | 0 .../model_config.yaml | 0 .../golden_values_lts.json | 0 .../golden_values_lts.json | 0 .../t5/t5_release/golden_values_0.9.0.json | 0 .../t5/t5_release/model_config.yaml | 0 tests/test_utils/python_scripts/common.py | 0 .../generate_jet_trigger_job.py | 0 .../python_scripts/generate_local_jobs.py | 0 .../python_scripts/launch_jet_workload.py | 0 .../test_utils/recipes/_build-mcore-dev.yaml | 0 .../test_utils/recipes/_build-mcore-lts.yaml | 0 tests/test_utils/recipes/_build-nemo.yaml | 0 tests/test_utils/recipes/bert.yaml | 0 tests/test_utils/recipes/gpt-modelopt.yaml | 0 tests/test_utils/recipes/gpt-nemo.yaml | 0 tests/test_utils/recipes/gpt.yaml | 0 .../test_utils/recipes/multimodal-llava.yaml | 2 + tests/test_utils/recipes/t5.yaml | 0 tests/test_utils/recipes/unit-tests.yaml | 0 tests/test_utils/shell_scripts/notify.sh | 0 tests/unit_tests/__init__.py | 0 tests/unit_tests/conftest.py | 0 tests/unit_tests/data/__init__.py | 0 tests/unit_tests/data/test_bin_reader.py | 0 tests/unit_tests/data/test_builder.py | 0 tests/unit_tests/data/test_gpt_dataset.py | 0 .../data/test_multimodal_dataset.py | 0 tests/unit_tests/data/test_preprocess_data.py | 0 .../unit_tests/data/test_preprocess_mmdata.py | 0 .../unit_tests/dist_checkpointing/__init__.py | 0 .../unit_tests/dist_checkpointing/conftest.py | 0 .../dist_checkpointing/models/__init__.py | 0 .../dist_checkpointing/models/common.py | 0 .../models/test_bert_model.py | 0 .../models/test_gpt_model.py | 0 .../dist_checkpointing/models/test_mamba.py | 0 .../dist_checkpointing/models/test_mlp_glu.py | 0 .../models/test_moe_experts.py | 20 +- .../models/test_retro_model.py | 0 .../models/test_t5_model.py | 0 .../dist_checkpointing/test_async_save.py | 0 .../test_cached_metadata.py | 0 .../test_flattened_resharding.py | 68 ++ .../unit_tests/dist_checkpointing/test_fp8.py | 0 .../dist_checkpointing/test_fully_parallel.py | 0 .../dist_checkpointing/test_local.py | 0 .../dist_checkpointing/test_mapping.py | 0 .../dist_checkpointing/test_nonpersistent.py | 0 .../dist_checkpointing/test_optimizer.py | 0 .../dist_checkpointing/test_serialization.py | 33 + tests/unit_tests/dist_checkpointing/utils.py | 0 ...est_grad_reduce_for_replicated_embedder.py | 0 .../distributed/test_param_and_grad_buffer.py | 0 tests/unit_tests/export/trtllm/__init__.py | 0 .../export/trtllm/test_distributed_fp8.py | 0 .../export/trtllm/test_single_device_fp8.py | 0 .../test_trtllm_distributed_gpu_converter.py | 0 .../export/trtllm/test_trtllm_helper.py | 0 .../export/trtllm/test_trtllm_layers.py | 0 .../test_trtllm_single_device_converter.py | 0 .../unit_tests/fusions/test_torch_softmax.py | 0 tests/unit_tests/inference/__init__.py | 0 .../unit_tests/inference/engines/__init__.py | 0 .../inference/engines/test_mcore_engine.py | 14 +- .../model_inference_wrappers/__init__.py | 0 .../gpt/test_gpt_inference_wrapper.py | 0 .../t5/test_t5_inference_wrapper.py | 0 .../test_model_inference_wrapper_config.py | 0 .../inference/test_common_inference_params.py | 6 +- .../unit_tests/inference/test_flash_decode.py | 0 .../inference/test_inference_utils.py | 0 .../inference/test_modelopt_gpt_model.py | 0 tests/unit_tests/inference/test_scheduler.py | 4 +- .../text_generation_controllers/__init__.py | 0 ...oder_decoder_text_generation_controller.py | 4 +- .../test_simple_text_generation_controller.py | 26 +- tests/unit_tests/models/__init__.py | 0 .../unit_tests/models/test_base_embedding.py | 0 tests/unit_tests/models/test_bert_model.py | 0 .../unit_tests/models/test_clip_vit_model.py | 0 tests/unit_tests/models/test_gpt_model.py | 0 tests/unit_tests/models/test_llava_model.py | 0 tests/unit_tests/models/test_mamba_model.py | 0 .../models/test_multimodal_projector.py | 0 tests/unit_tests/models/test_t5_model.py | 0 .../unit_tests/pipeline_parallel/__init__.py | 0 .../pipeline_parallel/test_helpers.py | 0 .../pipeline_parallel/test_schedules.py | 0 tests/unit_tests/ssm/test_mamba_block.py | 0 .../ssm/test_mamba_hybrid_layer_allocation.py | 0 tests/unit_tests/ssm/test_mamba_layer.py | 0 tests/unit_tests/ssm/test_mamba_mixer.py | 0 tests/unit_tests/tensor_parallel/__init__.py | 0 .../tensor_parallel/test_cross_entropy.py | 0 tests/unit_tests/tensor_parallel/test_data.py | 0 .../tensor_parallel/test_initialization.py | 0 .../unit_tests/tensor_parallel/test_layers.py | 0 .../tensor_parallel/test_mappings.py | 0 .../unit_tests/tensor_parallel/test_random.py | 0 .../test_tensor_parallel_utils.py | 0 tests/unit_tests/test_basic.py | 0 tests/unit_tests/test_imports.py | 0 tests/unit_tests/test_inference.py | 0 .../unit_tests/test_local_multi_tensor_fns.py | 0 .../test_num_microbatches_calculator.py | 0 tests/unit_tests/test_optimizer.py | 47 + .../test_optimizer_param_scheduler.py | 0 tests/unit_tests/test_parallel_state.py | 0 tests/unit_tests/test_tokenizer.py | 0 tests/unit_tests/test_training.py | 0 tests/unit_tests/test_utilities.py | 0 tests/unit_tests/test_utils.py | 0 tests/unit_tests/transformer/__init__.py | 0 tests/unit_tests/transformer/moe/__init__.py | 0 tests/unit_tests/transformer/moe/conftest.py | 0 .../moe/test_a2a_token_dispatcher.py | 0 .../transformer/moe/test_aux_loss.py | 44 + .../transformer/moe/test_grouped_mlp.py | 13 +- .../transformer/moe/test_moe_layer.py | 12 +- .../transformer/moe/test_routers.py | 63 +- .../transformer/moe/test_sequential_mlp.py | 4 +- .../transformer/moe/test_shared_experts.py | 6 +- .../transformer/moe/test_token_dispatcher.py | 4 +- .../transformer/moe/test_upcycling.py | 10 +- .../unit_tests/transformer/test_attention.py | 0 .../transformer/test_attention_packed_seq.py | 0 .../transformer/test_core_attention.py | 0 tests/unit_tests/transformer/test_mlp.py | 0 tests/unit_tests/transformer/test_module.py | 0 .../test_multi_latent_attention.py | 0 .../transformer/test_retro_attention.py | 0 tests/unit_tests/transformer/test_rope.py | 0 .../transformer/test_spec_customization.py | 0 .../transformer/test_transformer_block.py | 0 .../transformer/test_transformer_layer.py | 0 tools/autoformat.sh | 0 tools/bert_embedding/__init__.py | 0 tools/bert_embedding/dataset.py | 0 tools/bert_embedding/embed.py | 0 tools/bert_embedding/external_libs.py | 0 tools/bert_embedding/huggingface.py | 0 tools/checkpoint/convert.py | 7 +- tools/checkpoint/hybrid_conversion.py | 0 tools/checkpoint/loader_llama_mistral.py | 2 +- tools/checkpoint/loader_mcore.py | 0 tools/checkpoint/loader_megatron.py | 0 tools/checkpoint/loader_mixtral_hf.py | 0 tools/checkpoint/saver_mcore.py | 0 tools/checkpoint/saver_megatron.py | 2 +- tools/checkpoint/schema_base.py | 0 tools/checkpoint/schema_mcore.py | 0 tools/checkpoint/utils.py | 0 tools/copyright.sh | 0 tools/linter.py | 0 tools/merge_datasets.py | 0 tools/openwebtext/README.md | 0 tools/openwebtext/add_id.py | 0 tools/openwebtext/blacklist_urls.py | 0 tools/openwebtext/cleanup_dataset.py | 0 tools/openwebtext/cleanup_fix_dataset.py | 0 tools/openwebtext/filter_ngrams.py | 0 tools/openwebtext/find_duplicates.py | 0 tools/openwebtext/group_duplicate_url.py | 0 tools/openwebtext/merge_jsons.py | 0 tools/openwebtext/remove_group_duplicates.py | 0 tools/preprocess_data.py | 0 tools/preprocess_data_nmt.py | 0 tools/preprocess_mmdata.py | 0 tools/report_theoretical_memory.py | 0 tools/retro/README.md | 0 tools/retro/build_db.md | 0 tools/retro/cli/__init__.py | 0 tools/retro/cli/__main__.py | 0 tools/retro/cli/cli.py | 0 tools/retro/config_utils.py | 0 tools/retro/docker/Dockerfile | 0 tools/retro/preprocess_data.py | 0 tools/retro/sft/README.md | 0 tools/retro/sft/dataset_conv.py | 0 tools/retro/sft/open_inst.sh | 0 tools/retro/sft/sft_retro.py | 0 tools/retro/sft/sft_retro_lm.sh | 0 tools/retro/text_generation/evaluate.py | 0 tools/retro/text_generation/metrics.py | 0 tools/retro/text_generation/retro_api.py | 0 tools/retro/text_generation/retro_generate.sh | 0 .../retro/text_generation/retro_generation.py | 0 .../text_generation/retro_text_generation.py | 0 tools/run_mamba_text_generation_server.py | 0 tools/run_text_generation_server.py | 0 tools/run_vlm_text_generation.py | 0 tools/text_generation_cli.py | 0 train_GPT-MOE_567B.sh | 207 ++++ train_GPT-MOE_567B_1nodes.sh | 206 ++++ train_mixtral_8x7B_1nodes.sh | 67 +- train_mixtral_8x7B_2nodes.sh | 210 +++++ unit-test-job-lts.yaml | 0 1111 files changed, 4478 insertions(+), 1831 deletions(-) mode change 100755 => 100644 CHANGELOG.md mode change 100755 => 100644 CODEOWNERS mode change 100755 => 100644 CONTRIBUTING.md mode change 100755 => 100644 Dockerfile.ci.dev mode change 100755 => 100644 Dockerfile.ci.lts mode change 100755 => 100644 Dockerfile.linting mode change 100755 => 100644 GPT_pretraining.sh mode change 100755 => 100644 LICENSE mode change 100755 => 100644 MANIFEST.in mode change 100755 => 100644 README.md.origin mode change 100755 => 100644 docs/llama_mistral.md mode change 100755 => 100644 docs/source/api-guide/context_parallel.rst mode change 100755 => 100644 docs/source/api-guide/datasets.rst mode change 100755 => 100644 docs/source/api-guide/dist_checkpointing.rst mode change 100755 => 100644 docs/source/api-guide/dist_checkpointing.strategies.rst mode change 100755 => 100644 docs/source/api-guide/dist_optimizer.md mode change 100755 => 100644 docs/source/api-guide/distributed.rst mode change 100755 => 100644 docs/source/api-guide/encoder_decoder_parallelism.rst mode change 100755 => 100644 docs/source/api-guide/fusions.rst mode change 100755 => 100644 docs/source/api-guide/index.rst mode change 100755 => 100644 docs/source/api-guide/models.bert.rst mode change 100755 => 100644 docs/source/api-guide/models.gpt.rst mode change 100755 => 100644 docs/source/api-guide/models.rst mode change 100755 => 100644 docs/source/api-guide/models.t5.rst mode change 100755 => 100644 docs/source/api-guide/moe.rst mode change 100755 => 100644 docs/source/api-guide/num_microbatches_calculator.rst mode change 100755 => 100644 docs/source/api-guide/optimizer_param_scheduler.rst mode change 100755 => 100644 docs/source/api-guide/pipeline_parallel.rst mode change 100755 => 100644 docs/source/api-guide/tensor_parallel.rst mode change 100755 => 100644 docs/source/api-guide/transformer.rst mode change 100755 => 100644 docs/source/images/context_parallel/CP_overview.png mode change 100755 => 100644 docs/source/images/context_parallel/CP_results.png mode change 100755 => 100644 docs/source/images/distrib_optimizer/data_flow.png mode change 100755 => 100644 docs/source/images/distrib_optimizer/sharding_scheme.png mode change 100755 => 100644 docs/source/images/moe/token_drop.png mode change 100755 => 100644 docs/source/index.rst mode change 100755 => 100644 docs/source/user-guide/index.rst mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/README.md mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/annotations/filter-selfgeneration.py mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/annotations/perspective_api_annotate.py mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/annotations/preprocess.sh mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/finetune_gpt.py mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/finetune_gpt_distributed-1.3b.sh mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/generate-1.3b.sh mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/generate_samples_gpt.py mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/perspective_api.py mode change 100755 => 100644 examples/academic_paper_scripts/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh mode change 100755 => 100644 examples/academic_paper_scripts/msdp/README.md mode change 100755 => 100644 examples/academic_paper_scripts/msdp/data_processing.sh mode change 100755 => 100644 examples/academic_paper_scripts/msdp/eval_knwl_generation.sh mode change 100755 => 100644 examples/academic_paper_scripts/msdp/eval_resp_generation.sh mode change 100755 => 100644 examples/academic_paper_scripts/msdp/prep_resp_gen.sh mode change 100755 => 100644 examples/academic_paper_scripts/msdp/prompt_knwl_gen.sh mode change 100755 => 100644 examples/academic_paper_scripts/msdp/prompt_resp_gen.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/CONFIG.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/README.md mode change 100755 => 100644 examples/academic_paper_scripts/sc21/SBATCH.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/SRUN.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_11.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_12.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_13.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_14.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_15.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_16.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_17.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_figure_18.sh mode change 100755 => 100644 examples/academic_paper_scripts/sc21/run_table_1.sh mode change 100755 => 100644 examples/bert/README.md mode change 100755 => 100644 examples/bert/train_bert_340m_distributed.sh mode change 100755 => 100644 examples/export/README.md mode change 100755 => 100644 examples/export/knowledge_distillation/pretrain_gpt_modelopt.py mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/README.md mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/ptq_trtllm_llama2_7b.sh mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/ptq_trtllm_llama3_1_8b.sh mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/ptq_trtllm_llama3_8b.sh mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/ptq_trtllm_minitron_8b.sh mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/ptq_trtllm_mistral_12b.sh mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/ptq_trtllm_mixtral_8x7b.sh mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/text_generation_ptq.py mode change 100755 => 100644 examples/export/ptq_and_trtllm_export/trtllm_text_generation.py mode change 100755 => 100644 examples/export/trtllm_export/README.md mode change 100755 => 100644 examples/export/trtllm_export/distributed_export/gpt_distributed_gpu_export.py mode change 100755 => 100644 examples/export/trtllm_export/single_device_export/gpt_single_device_cpu_export.py mode change 100755 => 100644 examples/gpt3/README.md mode change 100755 => 100644 examples/gpt3/gpt_config.yaml mode change 100755 => 100644 examples/gpt3/train_gpt3_175b_distributed.sh mode change 100755 => 100644 examples/inference/README.md rename examples/inference/gpt/{simple_gpt_batch_inference.py => gpt_batch_inference.py} (91%) mode change 100755 => 100644 mode change 100755 => 100644 examples/inference/llama_mistral/huggingface_reference.py mode change 100755 => 100644 examples/inference/llama_mistral/run_text_generation_llama3.1.sh mode change 100755 => 100644 examples/inference/llama_mistral/run_text_generation_llama3.sh mode change 100755 => 100644 examples/inference/llama_mistral/run_text_generation_mistral.sh mode change 100755 => 100644 examples/inference/run_text_generation_server_345M.sh mode change 100755 => 100644 examples/inference/run_text_generation_server_345M_8_tensor_parallel.sh mode change 100755 => 100644 examples/inference/t5/simple_t5_batch_inference.py mode change 100755 => 100644 examples/mamba/.gitignore mode change 100755 => 100644 examples/mamba/Dockerfile mode change 100755 => 100644 examples/mamba/README.md mode change 100755 => 100644 examples/mamba/run_text_gen_server_8b.sh mode change 100755 => 100644 examples/mamba/run_text_gen_server_8b_gpt3.sh mode change 100755 => 100644 examples/mamba/train.sh mode change 100755 => 100644 examples/mixtral/README.md mode change 100755 => 100644 examples/mixtral/train_mixtral_8x7b_distributed.sh mode change 100755 => 100644 examples/multimodal/Dockerfile mode change 100755 => 100644 examples/multimodal/README.md mode change 100755 => 100644 examples/multimodal/assets/pretrain_curves.png mode change 100755 => 100644 examples/multimodal/combine_lm_vision_checkpoints.sh mode change 100755 => 100644 examples/multimodal/combine_state_dicts.py mode change 100755 => 100644 examples/multimodal/config.py mode change 100755 => 100644 examples/multimodal/convert_llava_pretrain_to_wds.py mode change 100755 => 100644 examples/multimodal/dataloader_provider.py mode change 100755 => 100644 examples/multimodal/dataset_helpers.py rename examples/multimodal/{ => evaluation}/evaluate_ai2d.py (72%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_chartqa.py (77%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_coco.py (77%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_mathvista.py (92%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_mmmu.py (91%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_ocrbench.py (95%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_textvqa.py (72%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluate_vqav2.py (88%) mode change 100755 => 100644 rename examples/multimodal/{ => evaluation}/evaluation_datasets.py (88%) mode change 100755 => 100644 mode change 100755 => 100644 examples/multimodal/image_processing.py mode change 100755 => 100644 examples/multimodal/layer_specs.py mode change 100755 => 100644 examples/multimodal/manual_prompts.json mode change 100755 => 100644 examples/multimodal/model.py mode change 100755 => 100644 examples/multimodal/model_converter/clip_converter.py mode change 100755 => 100644 examples/multimodal/model_converter/internvit_converter.py mode change 100755 => 100644 examples/multimodal/model_converter/siglip_converter.py mode change 100755 => 100644 examples/multimodal/model_converter/vision_model_tester.py mode change 100755 => 100644 examples/multimodal/multimodal_args.py mode change 100755 => 100644 examples/multimodal/nvlm/README.md mode change 100755 => 100644 examples/multimodal/nvlm/internvit.py mode change 100755 => 100644 examples/multimodal/nvlm/nvlm_prompts.json mode change 100755 => 100644 examples/multimodal/nvlm/pp_checkpoint_converter.py mode change 100755 => 100644 examples/multimodal/nvlm/pretrain_blend.yaml mode change 100755 => 100644 examples/multimodal/nvlm/pretrain_qwen20_72b_internvit_6b.sh mode change 100755 => 100644 examples/multimodal/nvlm/pretrain_yi_34b_internvit_6b.sh mode change 100755 => 100644 examples/multimodal/nvlm/run_text_generation_qwen20_72b_internvit_6b.sh create mode 100644 examples/multimodal/nvlm/run_text_generation_qwen25_7b_siglip.sh mode change 100755 => 100644 examples/multimodal/nvlm/run_text_generation_yi_34b_internvit_6b.sh mode change 100755 => 100644 examples/multimodal/nvlm/sft_34b_internvit.sh mode change 100755 => 100644 examples/multimodal/nvlm/sft_blend.yaml mode change 100755 => 100644 examples/multimodal/nvlm/sft_qwen20_72b_internvit_6b.sh mode change 100755 => 100644 examples/multimodal/pretrain_dataset.yaml mode change 100755 => 100644 examples/multimodal/pretrain_mistral_clip.sh mode change 100755 => 100644 examples/multimodal/run_text_generation.py mode change 100755 => 100644 examples/multimodal/sft_dataset.yaml mode change 100755 => 100644 examples/multimodal/sft_mistral_clip.sh mode change 100755 => 100644 examples/multimodal/text_generation_mistral_clip.sh mode change 100755 => 100644 examples/multimodal/train.py mode change 100755 => 100644 examples/retro/README.md mode change 100755 => 100644 examples/retro/preprocess_data.sh mode change 100755 => 100644 examples/retro/train_retro_2b_distributed.sh mode change 100755 => 100644 examples/run_simple_mcore_train_loop.py mode change 100755 => 100644 examples/t5/README.md mode change 100755 => 100644 examples/t5/t5_mcore_train_curve.png mode change 100755 => 100644 examples/t5/train_t5_220m_distributed.sh create mode 100644 hostfile_gptmoe create mode 100644 hostfile_mixtral8x7B mode change 100755 => 100644 images/model_table.png mode change 100755 => 100644 images/strong_scaling.png mode change 100755 => 100644 images/weak_scaling.png create mode 100644 kill.sh mode change 100755 => 100644 megatron/core/QuickStart.md mode change 100755 => 100644 megatron/core/README.md mode change 100755 => 100644 megatron/core/README_STRAGGLER.md mode change 100755 => 100644 megatron/core/__init__.py mode change 100755 => 100644 megatron/core/config_logger.py mode change 100755 => 100644 megatron/core/datasets/Makefile mode change 100755 => 100644 megatron/core/datasets/__init__.py mode change 100755 => 100644 megatron/core/datasets/bert_dataset.py mode change 100755 => 100644 megatron/core/datasets/blended_dataset.py mode change 100755 => 100644 megatron/core/datasets/blended_megatron_dataset_builder.py mode change 100755 => 100644 megatron/core/datasets/blended_megatron_dataset_config.py mode change 100755 => 100644 megatron/core/datasets/gpt_dataset.py mode change 100755 => 100644 megatron/core/datasets/helpers.cpp mode change 100755 => 100644 megatron/core/datasets/helpers.py create mode 100755 megatron/core/datasets/helpers_cpp.cpython-310-x86_64-linux-gnu.so_bak mode change 100755 => 100644 megatron/core/datasets/indexed_dataset.py mode change 100755 => 100644 megatron/core/datasets/masked_dataset.py mode change 100755 => 100644 megatron/core/datasets/megatron_dataset.py mode change 100755 => 100644 megatron/core/datasets/megatron_tokenizer.py mode change 100755 => 100644 megatron/core/datasets/multimodal_dataset.py mode change 100755 => 100644 megatron/core/datasets/readme.md mode change 100755 => 100644 megatron/core/datasets/retro/__init__.py mode change 100755 => 100644 megatron/core/datasets/retro/config/__init__.py mode change 100755 => 100644 megatron/core/datasets/retro/config/bert_embedders.py mode change 100755 => 100644 megatron/core/datasets/retro/config/config.py mode change 100755 => 100644 megatron/core/datasets/retro/config/gpt_chunk_datasets.py mode change 100755 => 100644 megatron/core/datasets/retro/config/tokenizers.py mode change 100755 => 100644 megatron/core/datasets/retro/db/__init__.py mode change 100755 => 100644 megatron/core/datasets/retro/db/build.py mode change 100755 => 100644 megatron/core/datasets/retro/db/dataset.py mode change 100755 => 100644 megatron/core/datasets/retro/db/utils.py mode change 100755 => 100644 megatron/core/datasets/retro/external_libs.py mode change 100755 => 100644 megatron/core/datasets/retro/index/__init__.py mode change 100755 => 100644 megatron/core/datasets/retro/index/build.py mode change 100755 => 100644 megatron/core/datasets/retro/index/factory.py mode change 100755 => 100644 megatron/core/datasets/retro/index/index.py mode change 100755 => 100644 megatron/core/datasets/retro/index/indexes/__init__.py mode change 100755 => 100644 megatron/core/datasets/retro/index/indexes/faiss_base.py mode change 100755 => 100644 megatron/core/datasets/retro/index/indexes/faiss_par_add.py mode change 100755 => 100644 megatron/core/datasets/retro/index/utils.py mode change 100755 => 100644 megatron/core/datasets/retro/index/validate.py mode change 100755 => 100644 megatron/core/datasets/retro/query/__init__.py mode change 100755 => 100644 megatron/core/datasets/retro/query/gpt_chunk_dataset.py mode change 100755 => 100644 megatron/core/datasets/retro/query/multi_split_gpt_dataset.py mode change 100755 => 100644 megatron/core/datasets/retro/query/query.py mode change 100755 => 100644 megatron/core/datasets/retro/query/retro_dataset.py mode change 100755 => 100644 megatron/core/datasets/retro/query/utils.py mode change 100755 => 100644 megatron/core/datasets/retro/utils.py mode change 100755 => 100644 megatron/core/datasets/t5_dataset.py mode change 100755 => 100644 megatron/core/datasets/utils.py mode change 100755 => 100644 megatron/core/datasets/utils_s3.py mode change 100755 => 100644 megatron/core/dist_checkpointing/__init__.py mode change 100755 => 100644 megatron/core/dist_checkpointing/core.py mode change 100755 => 100644 megatron/core/dist_checkpointing/dict_utils.py mode change 100755 => 100644 megatron/core/dist_checkpointing/exchange_utils.py mode change 100755 => 100644 megatron/core/dist_checkpointing/mapping.py mode change 100755 => 100644 megatron/core/dist_checkpointing/optimizer.py mode change 100755 => 100644 megatron/core/dist_checkpointing/serialization.py mode change 100755 => 100644 megatron/core/dist_checkpointing/state_dict_transformation.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/__init__.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/async_utils.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/base.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/common.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/filesystem_async.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/fully_parallel.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/resharding.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/state_dict_saver.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/tensorstore.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/torch.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/two_stage.py mode change 100755 => 100644 megatron/core/dist_checkpointing/strategies/zarr.py mode change 100755 => 100644 megatron/core/dist_checkpointing/utils.py mode change 100755 => 100644 megatron/core/dist_checkpointing/validation.py mode change 100755 => 100644 megatron/core/distributed/README.md mode change 100755 => 100644 megatron/core/distributed/__init__.py mode change 100755 => 100644 megatron/core/distributed/data_parallel_base.py mode change 100755 => 100644 megatron/core/distributed/distributed_data_parallel.py mode change 100755 => 100644 megatron/core/distributed/distributed_data_parallel_config.py mode change 100755 => 100644 megatron/core/distributed/finalize_model_grads.py mode change 100755 => 100644 megatron/core/distributed/param_and_grad_buffer.py mode change 100755 => 100644 megatron/core/distributed/torch_fully_sharded_data_parallel.py mode change 100755 => 100644 megatron/core/enums.py mode change 100755 => 100644 megatron/core/export/__init__.py mode change 100755 => 100644 megatron/core/export/data_type.py mode change 100755 => 100644 megatron/core/export/export_config.py mode change 100755 => 100644 megatron/core/export/model_type.py mode change 100755 => 100644 megatron/core/export/trtllm/__init__.py mode change 100755 => 100644 megatron/core/export/trtllm/engine_builder/__init__.py mode change 100755 => 100644 megatron/core/export/trtllm/engine_builder/trtllm_engine_builder.py mode change 100755 => 100644 megatron/core/export/trtllm/model_to_trllm_mapping/__init__.py mode change 100755 => 100644 megatron/core/export/trtllm/model_to_trllm_mapping/default_conversion_dict.py mode change 100755 => 100644 megatron/core/export/trtllm/trt_model_config.py mode change 100755 => 100644 megatron/core/export/trtllm/trt_model_type.py mode change 100755 => 100644 megatron/core/export/trtllm/trtllm_helper.py mode change 100755 => 100644 megatron/core/export/trtllm/trtllm_layers.py mode change 100755 => 100644 megatron/core/export/trtllm/trtllm_weights_converter/__init__.py mode change 100755 => 100644 megatron/core/export/trtllm/trtllm_weights_converter/distributed_trtllm_model_weights_converter.py mode change 100755 => 100644 megatron/core/export/trtllm/trtllm_weights_converter/single_device_trtllm_model_weights_converter.py mode change 100755 => 100644 megatron/core/extensions/__init__.py mode change 100755 => 100644 megatron/core/extensions/transformer_engine.py mode change 100755 => 100644 megatron/core/fusions/__init__.py mode change 100755 => 100644 megatron/core/fusions/fused_bias_dropout.py mode change 100755 => 100644 megatron/core/fusions/fused_bias_geglu.py mode change 100755 => 100644 megatron/core/fusions/fused_bias_gelu.py mode change 100755 => 100644 megatron/core/fusions/fused_bias_swiglu.py mode change 100755 => 100644 megatron/core/fusions/fused_cross_entropy.py mode change 100755 => 100644 megatron/core/fusions/fused_layer_norm.py mode change 100755 => 100644 megatron/core/fusions/fused_softmax.py mode change 100755 => 100644 megatron/core/inference/__init__.py mode change 100755 => 100644 megatron/core/inference/ammo_support/__init__.py mode change 100755 => 100644 megatron/core/inference/ammo_support/gpt/model_specs.py mode change 100755 => 100644 megatron/core/inference/ammo_support/gpt/state_dict_hooks.py mode change 100755 => 100644 megatron/core/inference/common_inference_params.py mode change 100755 => 100644 megatron/core/inference/communication_utils.py mode change 100755 => 100644 megatron/core/inference/engines/__init__.py mode change 100755 => 100644 megatron/core/inference/engines/abstract_engine.py mode change 100755 => 100644 megatron/core/inference/engines/mcore_engine.py mode change 100755 => 100644 megatron/core/inference/inference_request.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/__init__.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/gpt/__init__.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/inference_wrapper_config.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/t5/__init__.py mode change 100755 => 100644 megatron/core/inference/model_inference_wrappers/t5/t5_inference_wrapper.py mode change 100755 => 100644 megatron/core/inference/modelopt_support/__init__.py mode change 100755 => 100644 megatron/core/inference/modelopt_support/gpt/__init__.py mode change 100755 => 100644 megatron/core/inference/modelopt_support/gpt/model_specs.py mode change 100755 => 100644 megatron/core/inference/modelopt_support/gpt/state_dict_hooks.py create mode 100644 megatron/core/inference/sampling_params.py mode change 100755 => 100644 megatron/core/inference/scheduler.py mode change 100755 => 100644 megatron/core/inference/text_generation_controllers/__init__.py mode change 100755 => 100644 megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py mode change 100755 => 100644 megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py create mode 100644 megatron/core/inference/text_generation_controllers/text_generation_controller.py mode change 100755 => 100644 megatron/core/inference/utils.py mode change 100755 => 100644 megatron/core/inference_params.py mode change 100755 => 100644 megatron/core/jit.py mode change 100755 => 100644 megatron/core/model_parallel_config.py mode change 100755 => 100644 megatron/core/models/T5/__init__.py mode change 100755 => 100644 megatron/core/models/T5/t5_model.py mode change 100755 => 100644 megatron/core/models/T5/t5_spec.py mode change 100755 => 100644 megatron/core/models/__init__.py mode change 100755 => 100644 megatron/core/models/bert/__init__.py mode change 100755 => 100644 megatron/core/models/bert/bert_layer_specs.py mode change 100755 => 100644 megatron/core/models/bert/bert_lm_head.py mode change 100755 => 100644 megatron/core/models/bert/bert_model.py mode change 100755 => 100644 megatron/core/models/bert/pooler.py mode change 100755 => 100644 megatron/core/models/common/__init__.py mode change 100755 => 100644 megatron/core/models/common/embeddings/__init__.py mode change 100755 => 100644 megatron/core/models/common/embeddings/language_model_embedding.py mode change 100755 => 100644 megatron/core/models/common/embeddings/rope_utils.py mode change 100755 => 100644 megatron/core/models/common/embeddings/rotary_pos_embedding.py mode change 100755 => 100644 megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py mode change 100755 => 100644 megatron/core/models/common/language_module/__init__.py mode change 100755 => 100644 megatron/core/models/common/language_module/language_module.py mode change 100755 => 100644 megatron/core/models/common/vision_module/__init__.py mode change 100755 => 100644 megatron/core/models/common/vision_module/vision_module.py mode change 100755 => 100644 megatron/core/models/gpt/__init__.py mode change 100755 => 100644 megatron/core/models/gpt/gpt_layer_specs.py mode change 100755 => 100644 megatron/core/models/gpt/gpt_model.py create mode 100644 megatron/core/models/gpt/moe_module_specs.py mode change 100755 => 100644 megatron/core/models/mamba/__init__.py mode change 100755 => 100644 megatron/core/models/mamba/mamba_layer_specs.py mode change 100755 => 100644 megatron/core/models/mamba/mamba_model.py mode change 100755 => 100644 megatron/core/models/multimodal/__init__.py mode change 100755 => 100644 megatron/core/models/multimodal/llava_model.py mode change 100755 => 100644 megatron/core/models/multimodal/llava_spec.py mode change 100755 => 100644 megatron/core/models/retro/__init__.py mode change 100755 => 100644 megatron/core/models/retro/base_attention.py mode change 100755 => 100644 megatron/core/models/retro/config.py mode change 100755 => 100644 megatron/core/models/retro/decoder_attention.py mode change 100755 => 100644 megatron/core/models/retro/decoder_spec.py mode change 100755 => 100644 megatron/core/models/retro/encoder_attention.py mode change 100755 => 100644 megatron/core/models/retro/encoder_spec.py mode change 100755 => 100644 megatron/core/models/retro/model.py mode change 100755 => 100644 megatron/core/models/retro/utils.py mode change 100755 => 100644 megatron/core/models/vision/__init__.py mode change 100755 => 100644 megatron/core/models/vision/clip_vit_model.py mode change 100755 => 100644 megatron/core/models/vision/multimodal_projector.py mode change 100755 => 100644 megatron/core/models/vision/vit_layer_specs.py mode change 100755 => 100644 megatron/core/num_microbatches_calculator.py mode change 100755 => 100644 megatron/core/optimizer/__init__.py mode change 100755 => 100644 megatron/core/optimizer/clip_grads.py mode change 100755 => 100644 megatron/core/optimizer/distrib_optimizer.py mode change 100755 => 100644 megatron/core/optimizer/grad_scaler.py mode change 100755 => 100644 megatron/core/optimizer/optimizer.py mode change 100755 => 100644 megatron/core/optimizer/optimizer_config.py mode change 100755 => 100644 megatron/core/optimizer_param_scheduler.py mode change 100755 => 100644 megatron/core/package_info.py mode change 100755 => 100644 megatron/core/packed_seq_params.py mode change 100755 => 100644 megatron/core/parallel_state.py mode change 100755 => 100644 megatron/core/pipeline_parallel/__init__.py mode change 100755 => 100644 megatron/core/pipeline_parallel/p2p_communication.py mode change 100755 => 100644 megatron/core/pipeline_parallel/schedules.py mode change 100755 => 100644 megatron/core/requirements.txt mode change 100755 => 100644 megatron/core/rerun_state_machine.py mode change 100755 => 100644 megatron/core/ssm/__init__.py mode change 100755 => 100644 megatron/core/ssm/mamba_block.py mode change 100755 => 100644 megatron/core/ssm/mamba_hybrid_layer_allocation.py mode change 100755 => 100644 megatron/core/ssm/mamba_layer.py mode change 100755 => 100644 megatron/core/ssm/mamba_mixer.py mode change 100755 => 100644 megatron/core/ssm/triton_cache_manager.py mode change 100755 => 100644 megatron/core/tensor_parallel/__init__.py mode change 100755 => 100644 megatron/core/tensor_parallel/cross_entropy.py mode change 100755 => 100644 megatron/core/tensor_parallel/data.py mode change 100755 => 100644 megatron/core/tensor_parallel/layers.py mode change 100755 => 100644 megatron/core/tensor_parallel/mappings.py mode change 100755 => 100644 megatron/core/tensor_parallel/random.py mode change 100755 => 100644 megatron/core/tensor_parallel/utils.py mode change 100755 => 100644 megatron/core/timers.py mode change 100755 => 100644 megatron/core/transformer/__init__.py mode change 100755 => 100644 megatron/core/transformer/attention.py mode change 100755 => 100644 megatron/core/transformer/cuda_graphs.py mode change 100755 => 100644 megatron/core/transformer/custom_layers/__init__.py mode change 100755 => 100644 megatron/core/transformer/custom_layers/transformer_engine.py mode change 100755 => 100644 megatron/core/transformer/dot_product_attention.py mode change 100755 => 100644 megatron/core/transformer/enums.py mode change 100755 => 100644 megatron/core/transformer/identity_op.py mode change 100755 => 100644 megatron/core/transformer/mlp.py mode change 100755 => 100644 megatron/core/transformer/module.py mode change 100755 => 100644 megatron/core/transformer/moe/README.md mode change 100755 => 100644 megatron/core/transformer/moe/__init__.py mode change 100755 => 100644 megatron/core/transformer/moe/experts.py mode change 100755 => 100644 megatron/core/transformer/moe/grouped_gemm_util.py mode change 100755 => 100644 megatron/core/transformer/moe/legacy_a2a_token_dispatcher.py mode change 100755 => 100644 megatron/core/transformer/moe/moe_layer.py mode change 100755 => 100644 megatron/core/transformer/moe/moe_utils.py mode change 100755 => 100644 megatron/core/transformer/moe/router.py mode change 100755 => 100644 megatron/core/transformer/moe/shared_experts.py mode change 100755 => 100644 megatron/core/transformer/moe/token_dispatcher.py mode change 100755 => 100644 megatron/core/transformer/moe/upcycling_utils.py mode change 100755 => 100644 megatron/core/transformer/multi_latent_attention.py mode change 100755 => 100644 megatron/core/transformer/spec_utils.py mode change 100755 => 100644 megatron/core/transformer/torch_layer_norm.py mode change 100755 => 100644 megatron/core/transformer/torch_norm.py mode change 100755 => 100644 megatron/core/transformer/transformer_block.py mode change 100755 => 100644 megatron/core/transformer/transformer_config.py mode change 100755 => 100644 megatron/core/transformer/transformer_layer.py mode change 100755 => 100644 megatron/core/transformer/utils.py mode change 100755 => 100644 megatron/core/utils.py mode change 100755 => 100644 megatron/inference/__init__.py mode change 100755 => 100644 megatron/inference/algos/__init__.py mode change 100755 => 100644 megatron/inference/algos/distillation.py mode change 100755 => 100644 megatron/inference/arguments.py mode change 100755 => 100644 megatron/inference/checkpointing.py mode change 100755 => 100644 megatron/inference/docs/distillation.md mode change 100755 => 100644 megatron/inference/endpoints/common.py mode change 100755 => 100644 megatron/inference/endpoints/completions.py mode change 100755 => 100644 megatron/inference/gpt/__init__.py mode change 100755 => 100644 megatron/inference/gpt/loss_func.py mode change 100755 => 100644 megatron/inference/gpt/model_provider.py mode change 100755 => 100644 megatron/inference/static/index.html mode change 100755 => 100644 megatron/inference/text_generation/__init__.py mode change 100755 => 100644 megatron/inference/text_generation/api.py mode change 100755 => 100644 megatron/inference/text_generation/beam_utils.py mode change 100755 => 100644 megatron/inference/text_generation/communication.py mode change 100755 => 100644 megatron/inference/text_generation/forward_step.py mode change 100755 => 100644 megatron/inference/text_generation/generation.py mode change 100755 => 100644 megatron/inference/text_generation/sampling.py mode change 100755 => 100644 megatron/inference/text_generation/tokenization.py mode change 100755 => 100644 megatron/inference/text_generation_server.py mode change 100755 => 100644 megatron/legacy/data/__init__.py mode change 100755 => 100644 megatron/legacy/data/autoaugment.py mode change 100755 => 100644 megatron/legacy/data/biencoder_dataset_utils.py mode change 100755 => 100644 megatron/legacy/data/data_samplers.py mode change 100755 => 100644 megatron/legacy/data/dataset_utils.py mode change 100755 => 100644 megatron/legacy/data/ict_dataset.py mode change 100755 => 100644 megatron/legacy/data/image_folder.py mode change 100755 => 100644 megatron/legacy/data/multimodal_dataset.py mode change 100755 => 100644 megatron/legacy/data/orqa_wiki_dataset.py mode change 100755 => 100644 megatron/legacy/data/realm_dataset_utils.py mode change 100755 => 100644 megatron/legacy/data/realm_index.py mode change 100755 => 100644 megatron/legacy/data/vit_dataset.py mode change 100755 => 100644 megatron/legacy/fp16_deprecated/loss_scaler.py mode change 100755 => 100644 megatron/legacy/fused_kernels/__init__.py mode change 100755 => 100644 megatron/legacy/fused_kernels/compat.h mode change 100755 => 100644 megatron/legacy/fused_kernels/tests/__init__.py mode change 100755 => 100644 megatron/legacy/fused_kernels/tests/test_fused_kernels.py mode change 100755 => 100644 megatron/legacy/fused_kernels/type_shim.h mode change 100755 => 100644 megatron/legacy/indexer.py mode change 100755 => 100644 megatron/legacy/model/__init__.py mode change 100755 => 100644 megatron/legacy/model/bert_model.py mode change 100755 => 100644 megatron/legacy/model/biencoder_model.py mode change 100755 => 100644 megatron/legacy/model/classification.py mode change 100755 => 100644 megatron/legacy/model/enums.py mode change 100755 => 100644 megatron/legacy/model/fused_bias_gelu.py mode change 100755 => 100644 megatron/legacy/model/fused_layer_norm.py mode change 100755 => 100644 megatron/legacy/model/fused_softmax.py mode change 100755 => 100644 megatron/legacy/model/gpt_model.py mode change 100755 => 100644 megatron/legacy/model/language_model.py mode change 100755 => 100644 megatron/legacy/model/module.py mode change 100755 => 100644 megatron/legacy/model/multiple_choice.py mode change 100755 => 100644 megatron/legacy/model/realm_model.py mode change 100755 => 100644 megatron/legacy/model/rms_norm.py mode change 100755 => 100644 megatron/legacy/model/t5_model.py mode change 100755 => 100644 megatron/legacy/model/transformer.py mode change 100755 => 100644 megatron/legacy/model/utils.py mode change 100755 => 100644 megatron/legacy/model/vision/classification.py mode change 100755 => 100644 megatron/legacy/model/vision/dino.py mode change 100755 => 100644 megatron/legacy/model/vision/esvit_swin_backbone.py mode change 100755 => 100644 megatron/legacy/model/vision/inpainting.py mode change 100755 => 100644 megatron/legacy/model/vision/knn_monitor.py mode change 100755 => 100644 megatron/legacy/model/vision/mit_backbone.py mode change 100755 => 100644 megatron/legacy/model/vision/swin_backbone.py mode change 100755 => 100644 megatron/legacy/model/vision/utils.py mode change 100755 => 100644 megatron/legacy/model/vision/vit_backbone.py mode change 100755 => 100644 megatron/legacy/mpu/tests/__init__.py mode change 100755 => 100644 megatron/legacy/mpu/tests/commons.py mode change 100755 => 100644 megatron/legacy/mpu/tests/test_cross_entropy.py mode change 100755 => 100644 megatron/legacy/mpu/tests/test_data.py mode change 100755 => 100644 megatron/legacy/mpu/tests/test_initialize.py mode change 100755 => 100644 megatron/legacy/mpu/tests/test_layers.py mode change 100755 => 100644 megatron/legacy/mpu/tests/test_random.py mode change 100755 => 100644 megatron/training/__init__.py mode change 100755 => 100644 megatron/training/activations.py mode change 100755 => 100644 megatron/training/arguments.py mode change 100755 => 100644 megatron/training/async_utils.py mode change 100755 => 100644 megatron/training/checkpointing.py mode change 100755 => 100644 megatron/training/dist_signal_handler.py mode change 100755 => 100644 megatron/training/ft_integration.py mode change 100755 => 100644 megatron/training/global_vars.py mode change 100755 => 100644 megatron/training/initialize.py mode change 100755 => 100644 megatron/training/log_handler.py mode change 100755 => 100644 megatron/training/one_logger_utils.py mode change 100755 => 100644 megatron/training/theoretical_memory_usage.py mode change 100755 => 100644 megatron/training/tokenizer/__init__.py mode change 100755 => 100644 megatron/training/tokenizer/bert_tokenization.py mode change 100755 => 100644 megatron/training/tokenizer/gpt2_tokenization.py mode change 100755 => 100644 megatron/training/tokenizer/multimodal_tokenizer.py mode change 100755 => 100644 megatron/training/tokenizer/tokenizer.py mode change 100755 => 100644 megatron/training/training.py mode change 100755 => 100644 megatron/training/utils.py mode change 100755 => 100644 megatron/training/yaml_arguments.py mode change 100755 => 100644 mypy.ini mode change 100755 => 100644 pretrain_bert.py mode change 100755 => 100644 pretrain_gpt.py mode change 100755 => 100644 pretrain_ict.py mode change 100755 => 100644 pretrain_mamba.py mode change 100755 => 100644 pretrain_retro.py mode change 100755 => 100644 pretrain_t5.py mode change 100755 => 100644 pretrain_vision_classify.py mode change 100755 => 100644 pretrain_vision_dino.py mode change 100755 => 100644 pretrain_vision_inpaint.py mode change 100755 => 100644 pretrain_vlm.py mode change 100755 => 100644 pyproject.toml mode change 100755 => 100644 pytest.ini rename requirements/{pytorch:24.01 => pytorch_24.01}/requirements.txt (100%) mode change 100755 => 100644 rename requirements/{pytorch:24.07 => pytorch_24.07}/requirements.txt (100%) mode change 100755 => 100644 create mode 100644 run.sh create mode 100644 run_GPT-MOE.sh create mode 100644 run_GPT-MOE_1nodes.sh create mode 100644 run_mixtral8x7B_1nodes.sh create mode 100644 run_mixtral8x7B_4nodes.sh mode change 100755 => 100644 setup.py mode change 100755 => 100644 tasks/data_utils.py mode change 100755 => 100644 tasks/ensemble_classifier.py mode change 100755 => 100644 tasks/eval_utils.py mode change 100755 => 100644 tasks/finetune_utils.py mode change 100755 => 100644 tasks/glue/data.py mode change 100755 => 100644 tasks/glue/finetune.py mode change 100755 => 100644 tasks/glue/mnli.py mode change 100755 => 100644 tasks/glue/qqp.py mode change 100755 => 100644 tasks/main.py mode change 100755 => 100644 tasks/msdp/README.md mode change 100755 => 100644 tasks/msdp/evaluate.py mode change 100755 => 100644 tasks/msdp/main.py mode change 100755 => 100644 tasks/msdp/metrics.py mode change 100755 => 100644 tasks/msdp/preprocessing.py mode change 100755 => 100644 tasks/msdp/prompt.py mode change 100755 => 100644 tasks/orqa/README.md mode change 100755 => 100644 tasks/orqa/evaluate_orqa.py mode change 100755 => 100644 tasks/orqa/evaluate_utils.py mode change 100755 => 100644 tasks/orqa/supervised/data.py mode change 100755 => 100644 tasks/orqa/supervised/eval_utils.py mode change 100755 => 100644 tasks/orqa/supervised/finetune.py mode change 100755 => 100644 tasks/orqa/unsupervised/nq.py mode change 100755 => 100644 tasks/orqa/unsupervised/qa_utils.py mode change 100755 => 100644 tasks/orqa/unsupervised/tokenizers.py mode change 100755 => 100644 tasks/quantize/calibrate_gpt.py mode change 100755 => 100644 tasks/race/data.py mode change 100755 => 100644 tasks/race/finetune.py mode change 100755 => 100644 tasks/vision/classification/classification.py mode change 100755 => 100644 tasks/vision/classification/eval_utils.py mode change 100755 => 100644 tasks/vision/finetune_utils.py mode change 100755 => 100644 tasks/vision/main.py mode change 100755 => 100644 tasks/vision/segmentation/cityscapes.py mode change 100755 => 100644 tasks/vision/segmentation/data.py mode change 100755 => 100644 tasks/vision/segmentation/finetune_segformer.py mode change 100755 => 100644 tasks/vision/segmentation/finetune_setr.py mode change 100755 => 100644 tasks/vision/segmentation/metrics.py mode change 100755 => 100644 tasks/vision/segmentation/seg_heads.py mode change 100755 => 100644 tasks/vision/segmentation/seg_models.py mode change 100755 => 100644 tasks/vision/segmentation/transforms.py mode change 100755 => 100644 tasks/vision/segmentation/utils.py mode change 100755 => 100644 tasks/zeroshot_gpt/datasets.py mode change 100755 => 100644 tasks/zeroshot_gpt/detokenizer.py mode change 100755 => 100644 tasks/zeroshot_gpt/evaluate.py mode change 100755 => 100644 tests/__init__.py mode change 100755 => 100644 tests/functional_tests/__init__.py mode change 100755 => 100644 tests/functional_tests/python_test_utils/__init__.py mode change 100755 => 100644 tests/functional_tests/python_test_utils/common.py mode change 100755 => 100644 tests/functional_tests/python_test_utils/get_test_results_from_tensorboard_logs.py mode change 100755 => 100644 tests/functional_tests/python_test_utils/test_ci_pipeline.py mode change 100755 => 100644 tests/functional_tests/python_test_utils/test_fp8_ci_pipeline.py mode change 100755 => 100644 tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py mode change 100755 => 100644 tests/functional_tests/shell_test_utils/_run_training.sh mode change 100755 => 100644 tests/functional_tests/shell_test_utils/run_ci_test.sh mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_local_spec_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_local_spec_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_local_spec_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_resume_torch_dist_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_resume_torch_dist_local_spec_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_mr_tp2_pp2_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp2/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp2/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp4_pp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp4_pp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp1_pp2/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp1_pp2/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp4_pp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp4_pp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_release/golden_values_0.9.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/bert/bert_release/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/common/ckpt_converter/__main__.py mode change 100755 => 100644 tests/functional_tests/test_cases/common/ckpt_converter/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt-nemo/gpt3-nemo_126m_mr_mbs1_gbs8_mcore_te_tp2_pp4_vp3_seq_par_overlap_p2p_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt-nemo/gpt3-nemo_126m_mr_mbs4_gbs64_mcore_te_tp1_pp1_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release/golden_values_0.8.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release/golden_values_0.9.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_fsdp2_resume_torch_dist_te/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_fp16/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_fp16/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_fp16/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_resume_torch_dist/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_resume_torch_dist/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_resume_torch_dist/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4_resume_torch_dist/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_resume_torch_dist_te_4experts2parallel/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_resume_torch_dist_te_2experts/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1_resume_torch/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1_resume_torch_dist/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_dist_optimizer_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_dist_optimizer_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_dist_optimizer_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2_resume_torch/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2_resume_torch/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2_resume_torch/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_resume_torch/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_resume_torch/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_resume_torch/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_vp1_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_vp1_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_vp1_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_4experts/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_4experts/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_4experts/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_overlap_grad_reduce/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_overlap_grad_reduce/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_overlap_grad_reduce/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_resume_torch/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_resume_torch/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_resume_torch/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_no_mmap_bin_files_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_uniform_full_recompute_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_uniform_full_recompute_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp1_uniform_full_recompute_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_cp4_a2a_p2p_nondeterministic_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_cp4_a2a_p2p_nondeterministic_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_cp4_a2a_p2p_nondeterministic_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_interleaved_no_fusion_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_interleaved_no_fusion_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_interleaved_no_fusion_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_disable_bias_linear_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_disable_bias_linear_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_disable_bias_linear_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_resume_torch_dist_sequence_parallel_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_resume_torch_dist_swiglu_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_sequence_parallel_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_sequence_parallel_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_sequence_parallel_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_swiglu_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_swiglu_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_swiglu_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_untie_embeddings_and_outputs_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_untie_embeddings_and_outputs_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_untie_embeddings_and_outputs_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_calculate_per_token_loss_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_calculate_per_token_loss_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_calculate_per_token_loss_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_decoupled_lr_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_decoupled_lr_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_decoupled_lr_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_untied_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_untied_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_untied_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_dist_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_tunable_overlap_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_tunable_overlap_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_tunable_overlap_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_uneven_pipeline_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_uneven_pipeline_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp4_vp1_uneven_pipeline_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_cp2_nondeterministic_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_cp2_nondeterministic_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_cp2_nondeterministic_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_fsdp2_resume_torch_dist_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_multi_dist_optimizer_instances_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_multi_dist_optimizer_instances_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_multi_dist_optimizer_instances_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_groupedGEMM_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_top2router_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_dist_optimizer_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_dist_optimizer_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_dist_optimizer_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_top2router_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_top2router_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_top2router_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts_etp1_ep4_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts_etp1_ep4_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp1_te_8experts_etp1_ep4_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_cp2_nondeterministic_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_cp2_nondeterministic_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_cp2_nondeterministic_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_cross_entropy_loss_fusion_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_cross_entropy_loss_fusion_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_cross_entropy_loss_fusion_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_ddp_average_in_collective_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_ddp_average_in_collective_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_ddp_average_in_collective_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_defer_embedding_wgrad_compute_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_defer_embedding_wgrad_compute_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_defer_embedding_wgrad_compute_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_no_create_attention_mask_in_dataloader_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_no_create_attention_mask_in_dataloader_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_no_create_attention_mask_in_dataloader_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_no_mmap_bin_files_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_no_mmap_bin_files_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_no_mmap_bin_files_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_no_mmap_bin_files_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_qk_layernorm_test_mode_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_qk_layernorm_test_mode_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_qk_layernorm_test_mode_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_tp2_pp2_resume_torch_dist_uninstall_te_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_tp2_pp2_uninstall_te_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_te_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_te_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_te_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_te_tp2_pp2_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp1_pp4_vp1_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp1_pp4_vp1_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp1_pp4_vp1_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp1_pp4_vp1_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_mr_tp2_pp2_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/gpt/gpt3_nightly_mcore_te_tp2_pp1_modelopt_distill_resume/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x22b_tp2pp8ep8vpp1_release/golden_values_0.9.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x22b_tp2pp8ep8vpp1_release/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release/golden_values_0.8.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release/golden_values_0.9.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release_sm/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x7b_tp1pp4ep8vpp8_release/golden_values_0.9.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/mixtral/mixtral_8x7b_tp1pp4ep8vpp8_release/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp1_pp1_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp1_pp1_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp1_pp1_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp2_pp3_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp2_pp3_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp2_pp3_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_etp3_dgx_a100_1N7G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_etp3_dgx_a100_1N7G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_etp3_dgx_a100_1N7G/model_config.yaml create mode 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/golden_values_dev.json create mode 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/golden_values_lts.json create mode 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/model_config.yaml create mode 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dist_opt_dgx_a100_1N8G/golden_values_dev.json create mode 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dist_opt_dgx_a100_1N8G/golden_values_lts.json create mode 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dist_opt_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_resume_torch_etp3_dgx_a100_1N7G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp2_pp2_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp4_pp1_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp4_pp1_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp4_pp1_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_te_tp4_pp1_resume_torch_dist_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp2_pp2_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp2_pp2_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp2_pp2_resume_torch_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp4_pp1_dgx_a100_1N8G/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp4_pp1_dgx_a100_1N8G/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp4_pp1_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_mr_mcore_tp4_pp1_resume_torch_dist_dgx_a100_1N8G/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_vp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_vp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_vp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_vp1_resume_torch/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp2_pp1_vp1/golden_values_dev.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp2_pp1_vp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_tp2_pp1_vp1/model_config.yaml mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_weekly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_220m_weekly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/golden_values_lts.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_release/golden_values_0.9.0.json mode change 100755 => 100644 tests/functional_tests/test_cases/t5/t5_release/model_config.yaml mode change 100755 => 100644 tests/test_utils/python_scripts/common.py mode change 100755 => 100644 tests/test_utils/python_scripts/generate_jet_trigger_job.py mode change 100755 => 100644 tests/test_utils/python_scripts/generate_local_jobs.py mode change 100755 => 100644 tests/test_utils/python_scripts/launch_jet_workload.py mode change 100755 => 100644 tests/test_utils/recipes/_build-mcore-dev.yaml mode change 100755 => 100644 tests/test_utils/recipes/_build-mcore-lts.yaml mode change 100755 => 100644 tests/test_utils/recipes/_build-nemo.yaml mode change 100755 => 100644 tests/test_utils/recipes/bert.yaml mode change 100755 => 100644 tests/test_utils/recipes/gpt-modelopt.yaml mode change 100755 => 100644 tests/test_utils/recipes/gpt-nemo.yaml mode change 100755 => 100644 tests/test_utils/recipes/gpt.yaml mode change 100755 => 100644 tests/test_utils/recipes/multimodal-llava.yaml mode change 100755 => 100644 tests/test_utils/recipes/t5.yaml mode change 100755 => 100644 tests/test_utils/recipes/unit-tests.yaml mode change 100755 => 100644 tests/test_utils/shell_scripts/notify.sh mode change 100755 => 100644 tests/unit_tests/__init__.py mode change 100755 => 100644 tests/unit_tests/conftest.py mode change 100755 => 100644 tests/unit_tests/data/__init__.py mode change 100755 => 100644 tests/unit_tests/data/test_bin_reader.py mode change 100755 => 100644 tests/unit_tests/data/test_builder.py mode change 100755 => 100644 tests/unit_tests/data/test_gpt_dataset.py mode change 100755 => 100644 tests/unit_tests/data/test_multimodal_dataset.py mode change 100755 => 100644 tests/unit_tests/data/test_preprocess_data.py mode change 100755 => 100644 tests/unit_tests/data/test_preprocess_mmdata.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/__init__.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/conftest.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/__init__.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/common.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_bert_model.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_gpt_model.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_mamba.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_mlp_glu.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_moe_experts.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_retro_model.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/models/test_t5_model.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_async_save.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_cached_metadata.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_flattened_resharding.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_fp8.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_fully_parallel.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_local.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_mapping.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_nonpersistent.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_optimizer.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/test_serialization.py mode change 100755 => 100644 tests/unit_tests/dist_checkpointing/utils.py mode change 100755 => 100644 tests/unit_tests/distributed/test_grad_reduce_for_replicated_embedder.py mode change 100755 => 100644 tests/unit_tests/distributed/test_param_and_grad_buffer.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/__init__.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/test_distributed_fp8.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/test_single_device_fp8.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/test_trtllm_distributed_gpu_converter.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/test_trtllm_helper.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/test_trtllm_layers.py mode change 100755 => 100644 tests/unit_tests/export/trtllm/test_trtllm_single_device_converter.py mode change 100755 => 100644 tests/unit_tests/fusions/test_torch_softmax.py mode change 100755 => 100644 tests/unit_tests/inference/__init__.py mode change 100755 => 100644 tests/unit_tests/inference/engines/__init__.py mode change 100755 => 100644 tests/unit_tests/inference/engines/test_mcore_engine.py mode change 100755 => 100644 tests/unit_tests/inference/model_inference_wrappers/__init__.py mode change 100755 => 100644 tests/unit_tests/inference/model_inference_wrappers/gpt/test_gpt_inference_wrapper.py mode change 100755 => 100644 tests/unit_tests/inference/model_inference_wrappers/t5/test_t5_inference_wrapper.py mode change 100755 => 100644 tests/unit_tests/inference/model_inference_wrappers/test_model_inference_wrapper_config.py mode change 100755 => 100644 tests/unit_tests/inference/test_common_inference_params.py mode change 100755 => 100644 tests/unit_tests/inference/test_flash_decode.py mode change 100755 => 100644 tests/unit_tests/inference/test_inference_utils.py mode change 100755 => 100644 tests/unit_tests/inference/test_modelopt_gpt_model.py mode change 100755 => 100644 tests/unit_tests/inference/test_scheduler.py mode change 100755 => 100644 tests/unit_tests/inference/text_generation_controllers/__init__.py mode change 100755 => 100644 tests/unit_tests/inference/text_generation_controllers/test_encoder_decoder_text_generation_controller.py mode change 100755 => 100644 tests/unit_tests/inference/text_generation_controllers/test_simple_text_generation_controller.py mode change 100755 => 100644 tests/unit_tests/models/__init__.py mode change 100755 => 100644 tests/unit_tests/models/test_base_embedding.py mode change 100755 => 100644 tests/unit_tests/models/test_bert_model.py mode change 100755 => 100644 tests/unit_tests/models/test_clip_vit_model.py mode change 100755 => 100644 tests/unit_tests/models/test_gpt_model.py mode change 100755 => 100644 tests/unit_tests/models/test_llava_model.py mode change 100755 => 100644 tests/unit_tests/models/test_mamba_model.py mode change 100755 => 100644 tests/unit_tests/models/test_multimodal_projector.py mode change 100755 => 100644 tests/unit_tests/models/test_t5_model.py mode change 100755 => 100644 tests/unit_tests/pipeline_parallel/__init__.py mode change 100755 => 100644 tests/unit_tests/pipeline_parallel/test_helpers.py mode change 100755 => 100644 tests/unit_tests/pipeline_parallel/test_schedules.py mode change 100755 => 100644 tests/unit_tests/ssm/test_mamba_block.py mode change 100755 => 100644 tests/unit_tests/ssm/test_mamba_hybrid_layer_allocation.py mode change 100755 => 100644 tests/unit_tests/ssm/test_mamba_layer.py mode change 100755 => 100644 tests/unit_tests/ssm/test_mamba_mixer.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/__init__.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_cross_entropy.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_data.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_initialization.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_layers.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_mappings.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_random.py mode change 100755 => 100644 tests/unit_tests/tensor_parallel/test_tensor_parallel_utils.py mode change 100755 => 100644 tests/unit_tests/test_basic.py mode change 100755 => 100644 tests/unit_tests/test_imports.py mode change 100755 => 100644 tests/unit_tests/test_inference.py mode change 100755 => 100644 tests/unit_tests/test_local_multi_tensor_fns.py mode change 100755 => 100644 tests/unit_tests/test_num_microbatches_calculator.py mode change 100755 => 100644 tests/unit_tests/test_optimizer.py mode change 100755 => 100644 tests/unit_tests/test_optimizer_param_scheduler.py mode change 100755 => 100644 tests/unit_tests/test_parallel_state.py mode change 100755 => 100644 tests/unit_tests/test_tokenizer.py mode change 100755 => 100644 tests/unit_tests/test_training.py mode change 100755 => 100644 tests/unit_tests/test_utilities.py mode change 100755 => 100644 tests/unit_tests/test_utils.py mode change 100755 => 100644 tests/unit_tests/transformer/__init__.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/__init__.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/conftest.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_a2a_token_dispatcher.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_aux_loss.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_grouped_mlp.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_moe_layer.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_routers.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_sequential_mlp.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_shared_experts.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_token_dispatcher.py mode change 100755 => 100644 tests/unit_tests/transformer/moe/test_upcycling.py mode change 100755 => 100644 tests/unit_tests/transformer/test_attention.py mode change 100755 => 100644 tests/unit_tests/transformer/test_attention_packed_seq.py mode change 100755 => 100644 tests/unit_tests/transformer/test_core_attention.py mode change 100755 => 100644 tests/unit_tests/transformer/test_mlp.py mode change 100755 => 100644 tests/unit_tests/transformer/test_module.py mode change 100755 => 100644 tests/unit_tests/transformer/test_multi_latent_attention.py mode change 100755 => 100644 tests/unit_tests/transformer/test_retro_attention.py mode change 100755 => 100644 tests/unit_tests/transformer/test_rope.py mode change 100755 => 100644 tests/unit_tests/transformer/test_spec_customization.py mode change 100755 => 100644 tests/unit_tests/transformer/test_transformer_block.py mode change 100755 => 100644 tests/unit_tests/transformer/test_transformer_layer.py mode change 100755 => 100644 tools/autoformat.sh mode change 100755 => 100644 tools/bert_embedding/__init__.py mode change 100755 => 100644 tools/bert_embedding/dataset.py mode change 100755 => 100644 tools/bert_embedding/embed.py mode change 100755 => 100644 tools/bert_embedding/external_libs.py mode change 100755 => 100644 tools/bert_embedding/huggingface.py mode change 100755 => 100644 tools/checkpoint/convert.py mode change 100755 => 100644 tools/checkpoint/hybrid_conversion.py mode change 100755 => 100644 tools/checkpoint/loader_llama_mistral.py mode change 100755 => 100644 tools/checkpoint/loader_mcore.py mode change 100755 => 100644 tools/checkpoint/loader_megatron.py mode change 100755 => 100644 tools/checkpoint/loader_mixtral_hf.py mode change 100755 => 100644 tools/checkpoint/saver_mcore.py mode change 100755 => 100644 tools/checkpoint/saver_megatron.py mode change 100755 => 100644 tools/checkpoint/schema_base.py mode change 100755 => 100644 tools/checkpoint/schema_mcore.py mode change 100755 => 100644 tools/checkpoint/utils.py mode change 100755 => 100644 tools/copyright.sh mode change 100755 => 100644 tools/linter.py mode change 100755 => 100644 tools/merge_datasets.py mode change 100755 => 100644 tools/openwebtext/README.md mode change 100755 => 100644 tools/openwebtext/add_id.py mode change 100755 => 100644 tools/openwebtext/blacklist_urls.py mode change 100755 => 100644 tools/openwebtext/cleanup_dataset.py mode change 100755 => 100644 tools/openwebtext/cleanup_fix_dataset.py mode change 100755 => 100644 tools/openwebtext/filter_ngrams.py mode change 100755 => 100644 tools/openwebtext/find_duplicates.py mode change 100755 => 100644 tools/openwebtext/group_duplicate_url.py mode change 100755 => 100644 tools/openwebtext/merge_jsons.py mode change 100755 => 100644 tools/openwebtext/remove_group_duplicates.py mode change 100755 => 100644 tools/preprocess_data.py mode change 100755 => 100644 tools/preprocess_data_nmt.py mode change 100755 => 100644 tools/preprocess_mmdata.py mode change 100755 => 100644 tools/report_theoretical_memory.py mode change 100755 => 100644 tools/retro/README.md mode change 100755 => 100644 tools/retro/build_db.md mode change 100755 => 100644 tools/retro/cli/__init__.py mode change 100755 => 100644 tools/retro/cli/__main__.py mode change 100755 => 100644 tools/retro/cli/cli.py mode change 100755 => 100644 tools/retro/config_utils.py mode change 100755 => 100644 tools/retro/docker/Dockerfile mode change 100755 => 100644 tools/retro/preprocess_data.py mode change 100755 => 100644 tools/retro/sft/README.md mode change 100755 => 100644 tools/retro/sft/dataset_conv.py mode change 100755 => 100644 tools/retro/sft/open_inst.sh mode change 100755 => 100644 tools/retro/sft/sft_retro.py mode change 100755 => 100644 tools/retro/sft/sft_retro_lm.sh mode change 100755 => 100644 tools/retro/text_generation/evaluate.py mode change 100755 => 100644 tools/retro/text_generation/metrics.py mode change 100755 => 100644 tools/retro/text_generation/retro_api.py mode change 100755 => 100644 tools/retro/text_generation/retro_generate.sh mode change 100755 => 100644 tools/retro/text_generation/retro_generation.py mode change 100755 => 100644 tools/retro/text_generation/retro_text_generation.py mode change 100755 => 100644 tools/run_mamba_text_generation_server.py mode change 100755 => 100644 tools/run_text_generation_server.py mode change 100755 => 100644 tools/run_vlm_text_generation.py mode change 100755 => 100644 tools/text_generation_cli.py create mode 100755 train_GPT-MOE_567B.sh create mode 100755 train_GPT-MOE_567B_1nodes.sh create mode 100755 train_mixtral_8x7B_2nodes.sh mode change 100755 => 100644 unit-test-job-lts.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md old mode 100755 new mode 100644 diff --git a/CODEOWNERS b/CODEOWNERS old mode 100755 new mode 100644 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md old mode 100755 new mode 100644 diff --git a/Dockerfile.ci.dev b/Dockerfile.ci.dev old mode 100755 new mode 100644 diff --git a/Dockerfile.ci.lts b/Dockerfile.ci.lts old mode 100755 new mode 100644 diff --git a/Dockerfile.linting b/Dockerfile.linting old mode 100755 new mode 100644 diff --git a/GPT_pretraining.sh b/GPT_pretraining.sh old mode 100755 new mode 100644 diff --git a/LICENSE b/LICENSE old mode 100755 new mode 100644 diff --git a/Llama_pretraining.sh b/Llama_pretraining.sh index 265e2b5..1ac8a73 100755 --- a/Llama_pretraining.sh +++ b/Llama_pretraining.sh @@ -1,79 +1,64 @@ #!/bin/bash set -eux - #export FLASH_ATTENTION_PRINT_PARAM=1 # Runs the "7B" parameter model export HSA_FORCE_FINE_GRAIN_PCIE=1 export OMP_NUM_THREADS=1 export NCCL_P2P_LEVEL=PXB # SYS - #export HIP_ALLOC_INITIALIZE=0 -#export GPU_MAX_HW_QUEUES=20 # sglang空泡 - +# export GPU_MAX_HW_QUEUES=10 export NCCL_ALGO=Ring export NCCL_NCHANNELS_PER_PEER=16 -export NCCL_MIN_NCHANNELS=20 +export NCCL_MIN_NCHANNELS=32 # 20 +export NCCL_MAX_NCHANNELS=32 # 20 export NCCL_IB_TIMEOUT=22 export CUDA_DEVICE_MAX_CONNECTIONS=1 - -export NCCL_IB_HCA=mlx5_1,mlx5_2 -# export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,,mlx5_4,,mlx5_5,,mlx5_6,,mlx5_7 -export NCCL_NET_GDR_LEVEL=SYS -export NCCL_NET_GDR_READ=0 +export NCCL_IB_HCA=mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1,mlx5_8:1,mlx5_9:1 +export NCCL_NET_GDR_LEVEL=7 +export NCCL_NET_GDR_READ=1 +export RCCL_SDMA_COPY_ENABLE=0 +export NCCL_TOPO_FILE="/public/home/xingjl/dependency/rccl-tests-0204/topo-input.xml" +# export NCCL_TOPO_FILE="/workspace/rccl-test/rccl-tests-0204/topo-input.xml" export GLOG_minloglevel=3 # 打印error级别的nccl日志 - -# export TORCH_COMPILE_DEBUG=1 # 查看编译后的图 - source /opt/dtk/env.sh # 导入hipblaslt库 # export LD_LIBRARY_PATH=/data/hipblaslt-install-0904/lib:$LD_LIBRARY_PATH +# export LD_LIBRARY_PATH=/data/hipblaslt-install-dtk-25.04-0212/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/public/home/xingjl/dependency/hipblaslt-install-0227/lib:$LD_LIBRARY_PATH # 更新rocblas # export LD_LIBRARY_PATH=/data/rocblas-install_qwen1211/lib:$LD_LIBRARY_PATH # export LD_LIBRARY_PATH=/data/rocblas-install_qwen1228/lib:$LD_LIBRARY_PATH -# export LD_LIBRARY_PATH=/data/rocblas-install_0107_trans/lib:$LD_LIBRARY_PATH - +export LD_LIBRARY_PATH=/public/home/xingjl/dependency/rocblas-install-0224/lib:$LD_LIBRARY_PATH +# export LD_LIBRARY_PATH=/data/rocblas-install-0118-bf16/lib:$LD_LIBRARY_PATH # torch控制多流转单流 -# export ALLREDUCE_STREAM_WITH_COMPUTE=1 - +export ALLREDUCE_STREAM_WITH_COMPUTE=1 +export SENDRECV_STREAM_WITH_COMPUTE=1 # prof采集添加同步, 避免卡顿 # export GPU_FLUSH_ON_EXECUTION=1 # export HIP_DIRECT_DISPATCH=0 - # 采集rocblas size -export ROCBLAS_LAYER=3 +# export ROCBLAS_LAYER=3 # 采集 fa size # export FLASH_ATTENTION_PRINT_PARAM=1 - -CHECKPOINT_PATH=./tmp_7b #$1 # +#增加编译缓存 +export cache_size_limit=64 +# CHECKPOINT_PATH=./Llama-2-7b-hf-to-meg-tp1-pp2 #CHECKPOINT_PATH=./tmp_7b # +SAVE_PATH=./tmp_7b TENSORBOARD_LOGS_PATH=./tmp_7b #$2 # -DATA_PATH="/data/datasets/nemo_pretrain/oscar-1GB/oscar-1GB-llama_text_document" - +DATA_PATH="/public/home/xingjl/megatron-lm/llama2_dataset/oscar-1GB_head-llama2_text_document" #_text_document +# DATA_PATH="/data/datasets/oscar-1GB-head/oscar-1GB_head-llama2_text_document" #_text_document GPT_MODEL_ARGS=( --num-layers 32 --hidden-size 4096 --ffn-hidden-size 11008 --num-attention-heads 32 --max-position-embeddings 4096 - --normalization RMSNorm - --position-embedding-type rope + --position-embedding-type rope # none # --untie-embeddings-and-output-weights # 分开处理embed和输出权重, 增加灵活性 ) - -# GPT_MODEL_ARGS=( -# --num-layers 40 -# --hidden-size 5120 -# --ffn-hidden-size 13824 -# --num-attention-heads 40 -# --max-position-embeddings 4096 - -# --normalization RMSNorm -# --position-embedding-type rope -# --untie-embeddings-and-output-weights # 分开处理embed和输出权重, 增加灵活性 -# ) - -# export NVTE_FLASH_ATTN=1 # 走cutlass -export NVTE_FLASH_ATTN_TRITON=1 # 走triton_fa +export NVTE_FLASH_ATTN=1 # 走cutlass +# export NVTE_FLASH_ATTN_TRITON=1 # 走triton_fa # --transformer-impl transformer_engine # 走core用这两组参数 # --use-mcore-models # --transformer-impl local # 走legacy用这两组参数 @@ -82,8 +67,8 @@ TRAINING_ARGS=( --transformer-impl local # 走legacy用这两组参数 --use-legacy-models --micro-batch-size 1 - --global-batch-size 64 #240 #60 #512 #64 - --train-iters 10 + --global-batch-size 64 #32 #240 #60 #512 #64 + --train-iters 50 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 @@ -96,7 +81,7 @@ TRAINING_ARGS=( --disable-bias-linear --attention-dropout 0 --hidden-dropout 0 - --no-gradient-accumulation-fusion # 开启后精度不对, apex更新后可以开启 + # --no-gradient-accumulation-fusion --swiglu --lr 3.0e-5 --lr-decay-style cosine @@ -109,37 +94,52 @@ TRAINING_ARGS=( # --recompute-method block --overlap-grad-reduce # 重叠ddp grad reduce # --tp-comm-overlap # tensor parallel comm和gemm重叠, 优化项未适配 - # --tp-comm-overlap-rs-dgrad # reduce-scatter和dgrad gemm重叠, 优化项未适配 + # --tp-comm-overlap-rs-dgrad # reduce-scatter和dgrad gemm重叠 --use-flash-attn-cutlass ) +# 使用torch fa的环境变量 +# export TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 +# export TORCHINDUCTOR_BENCHMARK_FUSION=1 +# export TORCHINDUCTOR_BENCHMARK_MULTI_TEMPLATES=1 +# export TORCHINDUCTOR_MAX_AUTOTUNE=1 +# export TORCHINDUCTOR_CACHE_DIR=./cache # --use-flash-attn-cutlass # cutlass fa # --use-flash-attn-triton # triton fa - +# --use-flash-attn-torch # torch fa MODEL_PARALLEL_ARGS=( --sequence-parallel - --tensor-model-parallel-size 2 + --tensor-model-parallel-size 1 --pipeline-model-parallel-size 2 + # --context-parallel-size 2 + # --num-layers-per-virtual-pipeline-stage 4 + # --microbatch-group-size-per-virtual-pipeline-stage 1 +# --no-overlap-p2p-communication # 开启后 ) - DATA_ARGS=( --data-path $DATA_PATH --seq-length 4096 #4096 --split 949,50,1 --tokenizer-type Llama2Tokenizer - --tokenizer-model /data/model_weights/llama2_7b_hf/tokenizer.model + --tokenizer-model /public/home/xingjl/megatron-lm/llama2_dataset/tokenizer.model + # --tokenizer-model /data/model_weights/llama2_7b_hf/tokenizer.model ) - EVAL_AND_LOGGING_ARGS=( --log-interval 1 --log-throughput --save-interval 1000 --eval-interval 1000 - --save $CHECKPOINT_PATH - --load $CHECKPOINT_PATH - --eval-iters 10 + #--save $SAVE_PATH + #--load $SAVE_PATH + --eval-iters 3 --tensorboard-dir $TENSORBOARD_LOGS_PATH ) - +# FINETUNE_ARGS=( +# # --finetune +# # --pretrained-checkpoint $CHECKPOINT_PATH +# --load $CHECKPOINT_PATH +# --no-load-optim +# --no-load-rng +# ) PROFILE_ARGS=( --profile --profile-step-start 4 @@ -148,20 +148,17 @@ PROFILE_ARGS=( --profile-ranks 0 1 2 3 4 5 6 7 --profile-dir prof_data ) - RANK=$OMPI_COMM_WORLD_RANK LOCAL_RANK=$OMPI_COMM_WORLD_LOCAL_RANK WORLD_SIZE=$OMPI_COMM_WORLD_SIZE DIST_URL=${1} -DIST_PORT=34567 - +DIST_PORT=34577 DISTRIBUTED_ARGS=( --rank ${RANK} --world-size ${WORLD_SIZE} --local-rank ${LOCAL_RANK} --dist-url tcp://${DIST_URL}:${DIST_PORT} ) - APP="python -u pretrain_gpt.py \ ${GPT_MODEL_ARGS[@]} \ ${TRAINING_ARGS[@]} \ @@ -169,53 +166,52 @@ APP="python -u pretrain_gpt.py \ ${DATA_ARGS[@]} \ ${EVAL_AND_LOGGING_ARGS[@]} \ ${DISTRIBUTED_ARGS[@]} \ - " # 开启profile # ${PROFILE_ARGS[@]} \ - +# export HIP_VISIBLE_DEVICES=0,7 # # 4,5,6,7 #, export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 # # 4,5,6,7 #, # export CUDA_VISIBLE_DEVICES=4,5,6,7 # 0,1,2,3, -${APP} -# case ${LOCAL_RANK} in -# [0]) -# # export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [1]) -# # export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [2]) -# # export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [3]) -# # export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [4]) -# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [5]) -# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [6]) -# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# [7]) -# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# ${APP} -# # numactl --cpunodebind=0 --membind=0 ${APP} -# ;; -# esac +# ${APP} +case ${LOCAL_RANK} in +[0]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + numactl --cpunodebind=0 --membind=0 ${APP} + ;; +[1]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + numactl --cpunodebind=1 --membind=1 ${APP} + ;; +[2]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + numactl --cpunodebind=2 --membind=2 ${APP} + ;; +[3]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + numactl --cpunodebind=3 --membind=3 ${APP} + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + ;; +[4]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + numactl --cpunodebind=4 --membind=4 ${APP} + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + ;; +[5]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + numactl --cpunodebind=5 --membind=5 ${APP} + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + ;; +[6]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + numactl --cpunodebind=6 --membind=6 ${APP} + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + ;; +[7]) + export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + numactl --cpunodebind=7 --membind=7 ${APP} + # hipprof --hip-trace --trace-off numactl --cpunodebind=0 --membind=0 ${APP} + ;; +esac \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in old mode 100755 new mode 100644 diff --git a/README.md.origin b/README.md.origin old mode 100755 new mode 100644 diff --git a/docs/llama_mistral.md b/docs/llama_mistral.md old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/context_parallel.rst b/docs/source/api-guide/context_parallel.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/datasets.rst b/docs/source/api-guide/datasets.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/dist_checkpointing.rst b/docs/source/api-guide/dist_checkpointing.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/dist_checkpointing.strategies.rst b/docs/source/api-guide/dist_checkpointing.strategies.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/dist_optimizer.md b/docs/source/api-guide/dist_optimizer.md old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/distributed.rst b/docs/source/api-guide/distributed.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/encoder_decoder_parallelism.rst b/docs/source/api-guide/encoder_decoder_parallelism.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/fusions.rst b/docs/source/api-guide/fusions.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/index.rst b/docs/source/api-guide/index.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/models.bert.rst b/docs/source/api-guide/models.bert.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/models.gpt.rst b/docs/source/api-guide/models.gpt.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/models.rst b/docs/source/api-guide/models.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/models.t5.rst b/docs/source/api-guide/models.t5.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/moe.rst b/docs/source/api-guide/moe.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/num_microbatches_calculator.rst b/docs/source/api-guide/num_microbatches_calculator.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/optimizer_param_scheduler.rst b/docs/source/api-guide/optimizer_param_scheduler.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/pipeline_parallel.rst b/docs/source/api-guide/pipeline_parallel.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/tensor_parallel.rst b/docs/source/api-guide/tensor_parallel.rst old mode 100755 new mode 100644 diff --git a/docs/source/api-guide/transformer.rst b/docs/source/api-guide/transformer.rst old mode 100755 new mode 100644 diff --git a/docs/source/images/context_parallel/CP_overview.png b/docs/source/images/context_parallel/CP_overview.png old mode 100755 new mode 100644 diff --git a/docs/source/images/context_parallel/CP_results.png b/docs/source/images/context_parallel/CP_results.png old mode 100755 new mode 100644 diff --git a/docs/source/images/distrib_optimizer/data_flow.png b/docs/source/images/distrib_optimizer/data_flow.png old mode 100755 new mode 100644 diff --git a/docs/source/images/distrib_optimizer/sharding_scheme.png b/docs/source/images/distrib_optimizer/sharding_scheme.png old mode 100755 new mode 100644 diff --git a/docs/source/images/moe/token_drop.png b/docs/source/images/moe/token_drop.png old mode 100755 new mode 100644 diff --git a/docs/source/index.rst b/docs/source/index.rst old mode 100755 new mode 100644 diff --git a/docs/source/user-guide/index.rst b/docs/source/user-guide/index.rst old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/README.md b/examples/academic_paper_scripts/detxoify_lm/README.md old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/annotations/filter-selfgeneration.py b/examples/academic_paper_scripts/detxoify_lm/annotations/filter-selfgeneration.py old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/annotations/perspective_api_annotate.py b/examples/academic_paper_scripts/detxoify_lm/annotations/perspective_api_annotate.py old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/annotations/preprocess.sh b/examples/academic_paper_scripts/detxoify_lm/annotations/preprocess.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/finetune_gpt.py b/examples/academic_paper_scripts/detxoify_lm/finetune_gpt.py old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/finetune_gpt_distributed-1.3b.sh b/examples/academic_paper_scripts/detxoify_lm/finetune_gpt_distributed-1.3b.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/generate-1.3b.sh b/examples/academic_paper_scripts/detxoify_lm/generate-1.3b.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/generate_samples_gpt.py b/examples/academic_paper_scripts/detxoify_lm/generate_samples_gpt.py old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/perspective_api.py b/examples/academic_paper_scripts/detxoify_lm/perspective_api.py old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh b/examples/academic_paper_scripts/detxoify_lm/self_generation/selfgenerate-1.3b-unconditional.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/README.md b/examples/academic_paper_scripts/msdp/README.md old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/data_processing.sh b/examples/academic_paper_scripts/msdp/data_processing.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/eval_knwl_generation.sh b/examples/academic_paper_scripts/msdp/eval_knwl_generation.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/eval_resp_generation.sh b/examples/academic_paper_scripts/msdp/eval_resp_generation.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/prep_resp_gen.sh b/examples/academic_paper_scripts/msdp/prep_resp_gen.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/prompt_knwl_gen.sh b/examples/academic_paper_scripts/msdp/prompt_knwl_gen.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/msdp/prompt_resp_gen.sh b/examples/academic_paper_scripts/msdp/prompt_resp_gen.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/CONFIG.sh b/examples/academic_paper_scripts/sc21/CONFIG.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/README.md b/examples/academic_paper_scripts/sc21/README.md old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/SBATCH.sh b/examples/academic_paper_scripts/sc21/SBATCH.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/SRUN.sh b/examples/academic_paper_scripts/sc21/SRUN.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_11.sh b/examples/academic_paper_scripts/sc21/run_figure_11.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_12.sh b/examples/academic_paper_scripts/sc21/run_figure_12.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_13.sh b/examples/academic_paper_scripts/sc21/run_figure_13.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_14.sh b/examples/academic_paper_scripts/sc21/run_figure_14.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_15.sh b/examples/academic_paper_scripts/sc21/run_figure_15.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_16.sh b/examples/academic_paper_scripts/sc21/run_figure_16.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_17.sh b/examples/academic_paper_scripts/sc21/run_figure_17.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_figure_18.sh b/examples/academic_paper_scripts/sc21/run_figure_18.sh old mode 100755 new mode 100644 diff --git a/examples/academic_paper_scripts/sc21/run_table_1.sh b/examples/academic_paper_scripts/sc21/run_table_1.sh old mode 100755 new mode 100644 diff --git a/examples/bert/README.md b/examples/bert/README.md old mode 100755 new mode 100644 diff --git a/examples/bert/train_bert_340m_distributed.sh b/examples/bert/train_bert_340m_distributed.sh old mode 100755 new mode 100644 diff --git a/examples/export/README.md b/examples/export/README.md old mode 100755 new mode 100644 diff --git a/examples/export/knowledge_distillation/pretrain_gpt_modelopt.py b/examples/export/knowledge_distillation/pretrain_gpt_modelopt.py old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/README.md b/examples/export/ptq_and_trtllm_export/README.md old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/ptq_trtllm_llama2_7b.sh b/examples/export/ptq_and_trtllm_export/ptq_trtllm_llama2_7b.sh old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/ptq_trtllm_llama3_1_8b.sh b/examples/export/ptq_and_trtllm_export/ptq_trtllm_llama3_1_8b.sh old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/ptq_trtllm_llama3_8b.sh b/examples/export/ptq_and_trtllm_export/ptq_trtllm_llama3_8b.sh old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/ptq_trtllm_minitron_8b.sh b/examples/export/ptq_and_trtllm_export/ptq_trtllm_minitron_8b.sh old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/ptq_trtllm_mistral_12b.sh b/examples/export/ptq_and_trtllm_export/ptq_trtllm_mistral_12b.sh old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/ptq_trtllm_mixtral_8x7b.sh b/examples/export/ptq_and_trtllm_export/ptq_trtllm_mixtral_8x7b.sh old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/text_generation_ptq.py b/examples/export/ptq_and_trtllm_export/text_generation_ptq.py old mode 100755 new mode 100644 diff --git a/examples/export/ptq_and_trtllm_export/trtllm_text_generation.py b/examples/export/ptq_and_trtllm_export/trtllm_text_generation.py old mode 100755 new mode 100644 diff --git a/examples/export/trtllm_export/README.md b/examples/export/trtllm_export/README.md old mode 100755 new mode 100644 diff --git a/examples/export/trtllm_export/distributed_export/gpt_distributed_gpu_export.py b/examples/export/trtllm_export/distributed_export/gpt_distributed_gpu_export.py old mode 100755 new mode 100644 diff --git a/examples/export/trtllm_export/single_device_export/gpt_single_device_cpu_export.py b/examples/export/trtllm_export/single_device_export/gpt_single_device_cpu_export.py old mode 100755 new mode 100644 diff --git a/examples/gpt3/README.md b/examples/gpt3/README.md old mode 100755 new mode 100644 diff --git a/examples/gpt3/gpt_config.yaml b/examples/gpt3/gpt_config.yaml old mode 100755 new mode 100644 index 443e4b7..0625782 --- a/examples/gpt3/gpt_config.yaml +++ b/examples/gpt3/gpt_config.yaml @@ -63,6 +63,7 @@ language_model: # MoE related moe_router_load_balancing_type: "aux_loss" moe_router_topk: 2 + moe_router_topk_limited_devices: null moe_grouped_gemm: False moe_aux_loss_coeff: 0 # 1e-2 would be a good start value for load balance loss. moe_z_loss_coeff: null # 1e-3 would be a good start value for z-loss diff --git a/examples/gpt3/train_gpt3_175b_distributed.sh b/examples/gpt3/train_gpt3_175b_distributed.sh old mode 100755 new mode 100644 diff --git a/examples/inference/README.md b/examples/inference/README.md old mode 100755 new mode 100644 index bd8e738..b4b07cb --- a/examples/inference/README.md +++ b/examples/inference/README.md @@ -1,5 +1,5 @@ ### Megatron Core Inference Documentation -This guide will walk you through how you can use megatron core for inference on your models. +This guide provides an example for Megatron Core for running model inference. ### Contents - [Megatron Core Inference Documentation](#megatron-core-inference-documentation) @@ -18,21 +18,21 @@ This guide will walk you through how you can use megatron core for inference on
#### 1. Quick Start -This will walk you through the flow of running batch inference on a GPT model trained using megatron core. The file can be found at [simple_gpt_batch_inference.py](./gpt/simple_gpt_batch_inference.py) +This example runs batch inference on a GPT model trained using Megatron Core. The entrypoint is [simple_gpt_batch_inference.py](./gpt/gpt_batch_inference.py)
-##### 1.1 Understanding The Code -***STEP 1 - We initialize model parallel and other default arguments*** -We can default micro batch size to be 1, since for TP models it is not used, and for PP models it is calculated during runtime. +##### 1.1 Code Walkthrough +***STEP 1 - Initialize model parallel and other default arguments*** +The micro batch size is set as 1 as it is not used in tensor-parallelism only, and for pipeline-parallel models it is calculated at runtime. ```python initialize_megatron( args_defaults={'no_load_rng': True, 'no_load_optim': True, 'micro_batch_size': 1} ) ``` -***STEP 2 - We load the model using the model_provider_function*** -NOTE: The model provider function in the script supports MCore and Legacy models. +***STEP 2 - Load the model using the model_provider_function*** +NOTE: The model provider function supports both MCore and Legacy models. ```python model = get_model(model_provider, wrap_with_ddp=False) @@ -41,10 +41,10 @@ NOTE: The model provider function in the script supports MCore and Legacy models ``` ***STEP 3 - Choose an engine*** -One of the important elements of the generate function is an inference engine. In this example we will be choosing the [megatron core engine](../../megatron/core/inference/engine/mcore_engine.py) with a [simple text generation controller](../../megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py), the default engine. Other engines that will be supported in the future are TRTLLMEngine. +Text generation requires an inference engine, which includes a scheduler. The default engine is the [Megatron Core engine](../../megatron/core/inference/engine/mcore_engine.py) with a simple [text generation controller](../../megatron/core/inference/text_generation_controllers/text_generation_controller.py). TRTLLMEngine will be supported in the future. ```python inference_wrapped_model = GPTInferenceWrapper(model, args) - text_generation_controller = SimpleTextGenerationController( + text_generation_controller = TextGenerationController( inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer ) @@ -53,12 +53,12 @@ One of the important elements of the generate function is an inference engine. I ) ``` -***STEP 4 - Run the generate function and display results*** -We use default values for the [common inference params](../../megatron/core/inference/common_inference_params.py). Customize this if you want to change top_p, top_k, number of tokens to generate etc. -*Note that the result is returned as a list of [InferenceRequests](../../megatron/core/inference/inference_request.py)* +***STEP 4 - Run text generation*** +The [SamplingParams](../../megatron/core/inference/sampling_params.py) contains suggested defaults. Customize this to change top_p, top_k, number of tokens to generate etc. +*Note: The result is returned as a list of [InferenceRequests](../../megatron/core/inference/inference_request.py)* ```python results: List[InferenceRequest] = inference_engine.generate( - prompts=args.prompts, common_inference_params=common_inference_params + prompts=args.prompts, sampling_params=sampling_params ) if torch.distributed.get_rank() == 0: @@ -76,12 +76,12 @@ We use default values for the [common inference params](../../megatron/core/infe
##### 1.2 Running The Code -An example run script is shown below. Change the tokenizer paths, inference params, and other settings for your model. +An example run script is shown below. Set the tokenizer paths, inference params, and other settings appropriately. -For a quick recap on inference params refer to [this blog](https://ivibudh.medium.com/a-guide-to-controlling-llm-model-output-exploring-top-k-top-p-and-temperature-parameters-ed6a31313910) +For a quick recap on sampling parameters, refer to [this blog](https://ivibudh.medium.com/a-guide-to-controlling-llm-model-output-exploring-top-k-top-p-and-temperature-parameters-ed6a31313910). ``` -#In a slurm cluster (You could also use docker) +# In a slurm cluster (You could also use docker) ACCOUNT= MLM_PATH=/path/to/megatron-lm GPT_CKPT=/path/to/gpt/ckpt @@ -133,8 +133,8 @@ NOTE: Other parameters which can be customized for inference are :- --top_p (top_p sampling) --num-tokens-to-generate (Number of tokens to generate for each prompt) --inference-batch-times-seqlen-threshold (During inference, if batch-size times sequence-length is smaller than this threshold then we will not use pipelining, otherwise we will.') ---use-dist-ckpt (If you are using dist checkpoint format for the model) ---use-legacy-models (If you are using legacy gpt model instead of mcore gpt model) +--use-dist-ckpt (If using dist checkpoint format for the model) +--use-legacy-models (If using legacy gpt model instead of mcore gpt model) ``` @@ -142,16 +142,17 @@ NOTE: Other parameters which can be customized for inference are :-
-#### 2. Flow of Control In MCore Backend -The following is what happens in the [simple_gpt_batch_inference.py](./gpt/simple_gpt_batch_inference.py). -* We call [mcore_engine](../../megatron/core/inference/engines/mcore_engine.py) **generate()** function with all our input prompts. -* The scheduler in the engine will add these prompts to the [active requests] pool (../../megatron/core/inference/inference_request.py) until we hit the max batch size, and then it will put the rest in the waiting requests pool. -* The engine will then run until all requests (waiting + active) are completed +#### 2. Control Flow in the MCore Backend +An example of inference with static batching is provided in [gpt_batch_inference.py](./gpt/gpt_batch_inference.py). +* [mcore_engine](../../megatron/core/inference/engines/mcore_engine.py) **generate()** function is called with the input prompts. +* The `Scheduler` in the engine will add these prompts to the [active requests] pool (../../megatron/core/inference/inference_request.py) until max batch size is hit. Remaining requests will be added to the waiting requests pool. +* The engine will run until all requests (waiting + active) are completed. * The active requests are passed into **generate_all_output_tokens_static_batch()** of the text generation controller . - * This function uses the [model_inference_wrappers](../../megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py) **prep_model_for_inference()** , and then runs an auto regressive loop - * In the auto regressive loop, the **get_batch_for_context_window()** method of the inference wrapper is called to get the required input, passes it into the **run_one_forward_step()** method, which calls the appropriate (PP, TP) model `.forward()` methods to get the output logits - * The output logits are synchronized across all pipeline parallel ranks - * The text generation controller obtains the log probabilities and samples tokens based on the strategy defined in the common inference parameters. + * This function uses the **prep_model_for_inference()** method of the [model_inference_wrappers](../../megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py) and runs an autoregressive sampling loop + * In the autoregressive loop, the **get_batch_for_context_window()** method of the inference wrapper is called to slice out the input tokens and masks + * Input tokens and masks are passed it into the **run_one_forward_step()** method, which calls the model `.forward()` method to get the output logits + * Output logits are synchronized across all pipeline parallel ranks + * The text generation controller obtains the log probabilities and samples tokens based on the strategy defined in the sampling parameters. * The sampled tokens are then appended to the input prompt tokens for the next iteration * The **update_generation_status()** method of the text generation controller checks which prompts have finished generating or hit a stop condition * After the inference loop, the result is detokenized and stored as an attribute of the InferenceRequest. These requests are marked as completed. @@ -160,16 +161,18 @@ The following is what happens in the [simple_gpt_batch_inference.py](./gpt/simpl
#### 3. Customizing The Inference Pipeline -The following guide will walk you through how you can customize different parts of the inference pipeline. There are three levels at which you can customize the pipeline. -* **Inference engine** - Highest level of customization. Currently we support the MCore Engine. Change this to add a new engine. -* **Text generation controller** - Extend this to customize tokenization, detokenization, or implement a new sampling strategy. + +The inference pipeline supports three levels of customization: + +* **Inference engine** - The MCore Engine is currently supported. Change this to add a new backend. +* **Text generation controller** - The main sampling loop. This can be customized to support alternative tokenization, detokenization, or to implement a new sampling strategy. * **Inference Wrapped Model** - Change this to support a new model. * **Modify Inference Parameters** - Change this to update top_p, top_k, number of tokens to be generated, temperature, or other sampling parameters.
##### 3.1. Create Your Own Inference Backend -This is the highest level of customization. The [abstract_engine.py](./../../megatron/core/inference/engine/abstract_engine.py) file has a generate method that can be extended to support a new backend. +The [abstract_engine.py](./../../megatron/core/inference/engine/abstract_engine.py) file contains a `generate` method that can be extended to support a new backend. ```python class AbstractEngine(ABC): @@ -177,15 +180,17 @@ class AbstractEngine(ABC): def generate(self) -> dict: """The abstract backend's generate function. - To define your own backend, make sure you implement this and return the outputs as a dictionary . - + To define a new backend, implement this method and return the outputs as a dictionary. +```
-##### 3.2. Create Your Own Text Generation Controller -In case you want to use the megatron core backend, but would like to overwrite the tokenization, text generation or detokenization extend the [simple_text_generation_controller.py](../../megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py). The class has the following methods +##### 3.2. Implement a new Sampling Loop + +The [TextGenerationController](../../megatron/core/inference/text_generation_controllers/text_generation_controller.py) contains the main sampling loop and can be modified to support new tokenization, detokenization, or sampling strategies. + ``` python -class SimpleTextGenerationController: +class TextGenerationController: def tokenize_prompt(self, prompt: str) -> Tuple[torch.Tensor, torch.Tensor]: """Utility to tokenize the input prompts""" @@ -193,12 +198,12 @@ class SimpleTextGenerationController: def sample_from_logits( self, last_token_logits: torch.Tensor, - common_inference_params: CommonInferenceParams, + sampling_params: SamplingParams, vocab_size: int, ) -> torch.Tensor: """Samples the logits to generate outputs - Given the logits of the last token, this function samples it according to the parameters defined in common_inference_params and returns the samples + Given the logits of the last token, this function samples according to the parameters defined in sampling_params and returns the sampled tokens. """ def update_generation_status( @@ -229,12 +234,12 @@ class SimpleTextGenerationController:
##### 3.3. Support Other Models -In order to support other models please extend the [abstract_model_inference_wrapper.py](./../../megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py) file. The abstract wrapper already supports the following : -* Forward method which automatically calls the appropriate forward method (PP or TP etc) depending on model parallel settings -* Initalizes the model and puts it in eval mode -* Obtains the input parameters (batch size, max seq length) and has an instance of the input +Extend [abstract_model_inference_wrapper.py](./../../megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py) to support other models. The abstract model wrapper implements: +* Forward method which calls the model `forward` method depending on model parallel settings +* Initializes the model and puts it in `.eval()` mode +* Setup for the input parameters (max batch size, max seq length) -The main methods to change for your model might be the following: +The following methods should be implemented: ```python class AbstractModelInferenceWrapper: def prep_model_for_inference(self, prompts_tokens: torch.Tensor): @@ -247,28 +252,28 @@ class AbstractModelInferenceWrapper: def get_batch_for_context_window(self) -> List: """Returns the input data for inference - This function gets called iteratively in the inference loop . It can be used to extract relevant input from the prompt tokens, attention mask etc. required for each step in inference. + This function gets called iteratively in the inference loop. It can be used to extract relevant input from the prompt tokens, attention mask etc. required for each step in inference. ``` -Refer to [gpt_inference_wrapper.py](../../megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py) for an example of extending this for GPTModel. +Refer to [gpt_inference_wrapper.py](../../megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py) for an example of implementing this for GPTModel.
##### 3.3. Modify Inference Parameters -We use [common inference params](../../megatron/core/inference/common_inference_params.py) for text generation. Customize this if you want to change top_p, top_k, number of tokens to generate etc. If you want to add other attributes that you would use in the inference loop, you can do that as shown below +We use [common inference params](../../megatron/core/inference/sampling_params.py) for text generation. Customize this if you want to change top_p, top_k, number of tokens to generate etc. If you want to add other attributes that you would use in the inference loop, you can do that as shown below ``` -from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.sampling_params import SamplingParams -c = CommonInferenceParams(temperature=0.5) +c = SamplingParams(temperature=0.5) c.add_attributes({'min_length':4, 'eod_id':153}) ```
#### 4. Future work -The following are planned for the future releases . +The following features are planned for the future releases. * Dynamic batching * Paged Attention * TRTLLM Engine support -* Support for Multimodal model inference \ No newline at end of file +* Support for multimodal inference \ No newline at end of file diff --git a/examples/inference/gpt/simple_gpt_batch_inference.py b/examples/inference/gpt/gpt_batch_inference.py old mode 100755 new mode 100644 similarity index 91% rename from examples/inference/gpt/simple_gpt_batch_inference.py rename to examples/inference/gpt/gpt_batch_inference.py index 5c7ae5b..050b230 --- a/examples/inference/gpt/simple_gpt_batch_inference.py +++ b/examples/inference/gpt/gpt_batch_inference.py @@ -6,10 +6,10 @@ import sys from argparse import Namespace from megatron.core.inference.engines.abstract_engine import AbstractEngine from megatron.core.inference.engines.mcore_engine import MCoreEngine -from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import GPTInferenceWrapper from megatron.core.inference.inference_request import InferenceRequest -from megatron.core.inference.text_generation_controllers.simple_text_generation_controller import SimpleTextGenerationController +from megatron.core.inference.text_generation_controllers.text_generation_controller import TextGenerationController from megatron.core.transformer.module import MegatronModule sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))) @@ -66,7 +66,7 @@ def get_inference_engine(args: Namespace, model: MegatronModule) -> AbstractEngi ) inference_wrapped_model = GPTInferenceWrapper(model, inference_wrapper_config) - text_generation_controller = SimpleTextGenerationController(inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer) + text_generation_controller = TextGenerationController(inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer) return MCoreEngine(text_generation_controller=text_generation_controller, max_batch_size=args.max_batch_size) def main(): @@ -89,7 +89,7 @@ def main(): inference_engine = get_inference_engine(args, model) - common_inference_params = CommonInferenceParams( + sampling_params = SamplingParams( temperature=args.temperature, top_k=args.top_k, top_p=args.top_p, @@ -97,7 +97,7 @@ def main(): num_tokens_to_generate=args.num_tokens_to_generate) results: List[InferenceRequest] = inference_engine.generate( - prompts=args.prompts, common_inference_params=common_inference_params + prompts=args.prompts, sampling_params=sampling_params ) if torch.distributed.get_rank() == 0: diff --git a/examples/inference/llama_mistral/huggingface_reference.py b/examples/inference/llama_mistral/huggingface_reference.py old mode 100755 new mode 100644 diff --git a/examples/inference/llama_mistral/run_text_generation_llama3.1.sh b/examples/inference/llama_mistral/run_text_generation_llama3.1.sh old mode 100755 new mode 100644 diff --git a/examples/inference/llama_mistral/run_text_generation_llama3.sh b/examples/inference/llama_mistral/run_text_generation_llama3.sh old mode 100755 new mode 100644 diff --git a/examples/inference/llama_mistral/run_text_generation_mistral.sh b/examples/inference/llama_mistral/run_text_generation_mistral.sh old mode 100755 new mode 100644 diff --git a/examples/inference/run_text_generation_server_345M.sh b/examples/inference/run_text_generation_server_345M.sh old mode 100755 new mode 100644 diff --git a/examples/inference/run_text_generation_server_345M_8_tensor_parallel.sh b/examples/inference/run_text_generation_server_345M_8_tensor_parallel.sh old mode 100755 new mode 100644 diff --git a/examples/inference/t5/simple_t5_batch_inference.py b/examples/inference/t5/simple_t5_batch_inference.py old mode 100755 new mode 100644 index 3f4557d..b4226d7 --- a/examples/inference/t5/simple_t5_batch_inference.py +++ b/examples/inference/t5/simple_t5_batch_inference.py @@ -5,7 +5,7 @@ from argparse import Namespace import torch import pretrain_t5 -from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.engines.abstract_engine import AbstractEngine from megatron.core.inference.engines.mcore_engine import MCoreEngine from megatron.core.inference.inference_request import InferenceRequest @@ -120,7 +120,7 @@ def main(): inference_engine = get_inference_engine(args, model) - common_inference_params = CommonInferenceParams( + sampling_params = SamplingParams( temperature=args.temperature, top_k=args.top_k, top_p=args.top_p, @@ -138,7 +138,7 @@ def main(): prompts=args.prompts, add_BOS=True, encoder_prompts=args.encoder_prompts, - common_inference_params=common_inference_params, + sampling_params=sampling_params, ) if torch.distributed.get_rank() == 0: diff --git a/examples/mamba/.gitignore b/examples/mamba/.gitignore old mode 100755 new mode 100644 diff --git a/examples/mamba/Dockerfile b/examples/mamba/Dockerfile old mode 100755 new mode 100644 diff --git a/examples/mamba/README.md b/examples/mamba/README.md old mode 100755 new mode 100644 diff --git a/examples/mamba/run_text_gen_server_8b.sh b/examples/mamba/run_text_gen_server_8b.sh old mode 100755 new mode 100644 diff --git a/examples/mamba/run_text_gen_server_8b_gpt3.sh b/examples/mamba/run_text_gen_server_8b_gpt3.sh old mode 100755 new mode 100644 diff --git a/examples/mamba/train.sh b/examples/mamba/train.sh old mode 100755 new mode 100644 diff --git a/examples/mixtral/README.md b/examples/mixtral/README.md old mode 100755 new mode 100644 diff --git a/examples/mixtral/train_mixtral_8x7b_distributed.sh b/examples/mixtral/train_mixtral_8x7b_distributed.sh old mode 100755 new mode 100644 diff --git a/examples/multimodal/Dockerfile b/examples/multimodal/Dockerfile old mode 100755 new mode 100644 diff --git a/examples/multimodal/README.md b/examples/multimodal/README.md old mode 100755 new mode 100644 index 62e4756..a65839f --- a/examples/multimodal/README.md +++ b/examples/multimodal/README.md @@ -16,7 +16,7 @@ You can build a docker container using `examples/multimodal/Dockerfile` to run t ### Language model -Follow the instructions in [Mistral](../../docs/llama_mistral.md#mistral-7b) to download weights for Mistral-7B-Instruct-v0.3 (Base or Instruct) from HuggingFace and convert to mcore format with tensor parallel size 4. +Follow the instructions in [Mistral](../../docs/llama_mistral.md#mistral-7b) to download weights for Mistral-7B-Instruct-v0.3 from HuggingFace and convert to mcore format with tensor parallel size 4. Please use the tokenizer from HuggingFace. ### Vision model @@ -113,7 +113,7 @@ Run the following script: ``` examples/multimodal/text_generation_mistral_clip.sh --input-image-path /path/to/input/images --output-path /some/output/directory \ - --model-path /path/to/model.pt --tokenizer-path /path/to/tokenizer/ --gt-path /path/to/groundtruth/file --task generation-task-name + --model-path /path/to/model.pt --gt-path /path/to/groundtruth/file --task generation-task-name ``` where `--task generation-task-name` is the name of the evaluation benchmark such as `captioning` or `MMMU`. diff --git a/examples/multimodal/assets/pretrain_curves.png b/examples/multimodal/assets/pretrain_curves.png old mode 100755 new mode 100644 diff --git a/examples/multimodal/combine_lm_vision_checkpoints.sh b/examples/multimodal/combine_lm_vision_checkpoints.sh old mode 100755 new mode 100644 diff --git a/examples/multimodal/combine_state_dicts.py b/examples/multimodal/combine_state_dicts.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/config.py b/examples/multimodal/config.py old mode 100755 new mode 100644 index 343fcd5..ee40460 --- a/examples/multimodal/config.py +++ b/examples/multimodal/config.py @@ -7,34 +7,20 @@ from megatron.training.activations import fast_gelu, quick_gelu, squared_relu def get_language_model_config(config): - if config.language_model_type == "2b": + if config.language_model_type == "llama3_8b": + config.activation_func = torch.nn.functional.silu config.add_bias_linear = False config.bias_activation_fusion = False config.gated_linear_unit = True - config.apply_query_key_layer_scaling = True - config.layernorm_zero_centered_gamma = True - config.bias_dropout_fusion = False - config.rotary_percent = 0.5 - config.apply_rope_fusion = False - config.attention_softmax_in_fp32 = True - elif config.language_model_type == "8b": - config.add_bias_linear = False - config.bias_activation_fusion = False - config.gated_linear_unit = False - config.apply_query_key_layer_scaling = True - config.layernorm_zero_centered_gamma = True + config.apply_query_key_layer_scaling = False + config.layernorm_zero_centered_gamma = ( + False # Zero centered gamma not supported for RMSNorm + ) config.bias_dropout_fusion = False - config.rotary_percent = 0.5 - config.attention_dropout = 0.0 config.apply_rope_fusion = False - config.activation_func = squared_relu - config.ffn_hidden_size = 16384 - config.masked_softmax_fusion = True config.attention_softmax_in_fp32 = True - config.num_query_groups = 32 - config.kv_channels = 128 - config.rotary_interleaved = False - elif config.language_model_type == "llama3_8b": + config.ffn_hidden_size = 14336 + elif config.language_model_type == "mistral_7b": config.activation_func = torch.nn.functional.silu config.add_bias_linear = False config.bias_activation_fusion = False @@ -47,7 +33,7 @@ def get_language_model_config(config): config.apply_rope_fusion = False config.attention_softmax_in_fp32 = True config.ffn_hidden_size = 14336 - elif config.language_model_type == "mistral_7b": + elif config.language_model_type == "yi-34b": config.activation_func = torch.nn.functional.silu config.add_bias_linear = False config.bias_activation_fusion = False @@ -59,10 +45,11 @@ def get_language_model_config(config): config.bias_dropout_fusion = False config.apply_rope_fusion = False config.attention_softmax_in_fp32 = True - config.ffn_hidden_size = 14336 - elif config.language_model_type == "yi-34b": + config.ffn_hidden_size = 20480 + elif config.language_model_type == "qwen2.5_7B": config.activation_func = torch.nn.functional.silu config.add_bias_linear = False + config.add_qkv_bias = True config.bias_activation_fusion = False config.gated_linear_unit = True config.apply_query_key_layer_scaling = False @@ -72,7 +59,7 @@ def get_language_model_config(config): config.bias_dropout_fusion = False config.apply_rope_fusion = False config.attention_softmax_in_fp32 = True - config.ffn_hidden_size = 20480 + config.ffn_hidden_size = 18944 elif config.language_model_type == "qwen2.0_72B": config.activation_func = torch.nn.functional.silu config.add_bias_linear = False @@ -168,13 +155,7 @@ def get_vision_projection_config(config, hidden_size): config.bias_activation_fusion = False config.add_bias_linear = False config.hidden_size = hidden_size # Used as the vision projection output size, i.e., the input to the language model. - if config.language_model_type == "2b": - config.ffn_hidden_size = 5440 - config.activation_func = torch.nn.functional.gelu - if config.language_model_type == "8b": - config.ffn_hidden_size = 16384 - config.activation_func = squared_relu - elif config.language_model_type == "llama3_8b": + if config.language_model_type == "llama3_8b": config.ffn_hidden_size = 14336 config.activation_func = torch.nn.functional.gelu elif config.language_model_type == "mistral_7b": @@ -185,6 +166,9 @@ def get_vision_projection_config(config, hidden_size): config.ffn_hidden_size = 20480 config.normalization = "LayerNorm" config.activation_func = torch.nn.functional.gelu + elif config.language_model_type == "qwen2.5_7B": + config.ffn_hidden_size = 3584 + config.activation_func = torch.nn.functional.gelu elif config.language_model_type == "qwen2.0_72B": config.ffn_hidden_size = 29568 config.normalization = "LayerNorm" diff --git a/examples/multimodal/convert_llava_pretrain_to_wds.py b/examples/multimodal/convert_llava_pretrain_to_wds.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/dataloader_provider.py b/examples/multimodal/dataloader_provider.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/dataset_helpers.py b/examples/multimodal/dataset_helpers.py old mode 100755 new mode 100644 index de76f8e..ecbbc50 --- a/examples/multimodal/dataset_helpers.py +++ b/examples/multimodal/dataset_helpers.py @@ -2,16 +2,19 @@ import bisect import dataclasses import json +import re import sys import traceback from dataclasses import dataclass from typing import Dict, List, Optional, Tuple, Union from image_processing import get_visual_transform +from PIL import Image +from torchvision.transforms import ToPILImage import numpy as np import torch -from megatron.core.models.multimodal.llava_model import IGNORE_INDEX, IMAGE_TOKEN +from megatron.core.models.multimodal.llava_model import IGNORE_INDEX, IMAGE_TOKEN, VIDEO_TOKEN from megatron.core.models.vision.clip_vit_model import get_num_image_embeddings from megatron.energon import ( Batch, @@ -175,6 +178,10 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, self.img_h, self.img_w = self.args.img_h, self.args.img_w + # This map is used to reduce the number of tiles used per image if the number of tokens is + # larger than the decoder_seq_length. + self.num_tiles_degradation_map = {12:8, 8:6, 6:4, 4:2, 2:1, 1:1} + def _get_total_seq_length(self, input_ids, num_tiles): """Calculate expected sequence length given text tokens length and number of tiles.""" total_num_images = len(num_tiles) @@ -237,7 +244,7 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, prompt_idx = np.random.randint(len(prompt_list)) cur_prompt = prompt_list[prompt_idx] - cur_prompt = "\n" + cur_prompt + "\n" + cur_prompt = IMAGE_TOKEN + "\n" + cur_prompt + "\n" caption = sample.caption.strip() @@ -282,7 +289,7 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, # LLAVA training: override text-prompt with just the image. conv = [ # Note: no system message. - {"role": "user", "content": "\n"}, + {"role": "user", "content": IMAGE_TOKEN + "\n"}, {"role": "assistant", "content": sample.answers}, ] @@ -307,66 +314,130 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, """Encode SFT sample.""" augment = sample.__subflavors__['augmentation'] if 'augmentation' in sample.__subflavors__ else False has_video = sample.__subflavors__['has_video'] if 'has_video' in sample.__subflavors__ else False - has_image = sample.__subflavors__['has_image'] if 'has_image' in sample.__subflavors__ else False - has_image = has_image or (hasattr(sample, "images") and len(sample.images) > 0) - if has_video: - # Grab the selected frames of the video as a tensor with shape - # fhwc: (num_frames, height, width, num_channels). - video_fhwc = sample.images[0].permute(0, 2, 3, 1) - selected_frames = torch.linspace( - 0, video_fhwc.shape[0] - 1, self.args.num_frames).long() - video_frame_fhwc = video_fhwc[selected_frames] - imgs = [] - for video_frame_hwc in video_frame_fhwc: - imgs += get_visual_transform( - video_frame_hwc, self.img_h, self.img_w, - self.args.use_tiling, self.args.max_num_tiles, - self.args.use_thumbnail, augment, self.args.vision_model_type) - num_tiles = [len(imgs)] - elif has_image: - imgs = get_visual_transform( - sample.images[0], self.img_h, self.img_w, self.args.use_tiling, self.args.max_num_tiles, self.args.use_thumbnail, augment, - self.args.vision_model_type, - ) - num_tiles = [len(imgs)] - else: - imgs = num_tiles = [] - sample.__key__ = "{}-{}".format("no-image", sample.__key__) + has_image = False + if hasattr(sample, "images"): + # If this is a text-only sample and we are freezing the LM, + # then use a dummy input image. + if len(sample.images) == 0 and self.args.freeze_LM: + empty_img = Image.new('RGB', (self.args.img_w, self.args.img_h), (255, 255, 255)) + sample.images.append(empty_img) + if len(sample.images) > 0 and not has_video: + has_image = True - conversation = [] # Note: Some tokenizers may ignore the system prompt. - conversation.append({"role": "system", "content": "Answer the questions."}) - - has_image_token = False - + conversation = [{"role": "system", "content": "Answer the questions."}] + # Format the conversation as a list of "user" / "assistant" turns. for text in sample.texts: - if IMAGE_TOKEN in text["value"]: - has_image_token = True - - if text["from"] == "human": - role = "user" - elif text["from"] == "gpt": - role = "assistant" - else: - raise RuntimeError(f"unexpected role {text['from']} in {sample.texts}") - - turn = {"role": role, "content": text["value"]} - conversation.append(turn) - - # If the sample contains an image but none of the user messages has an image token, - # then add it to the first user message. - if len(imgs) > 0 and not has_image_token: + error_msg = f"unexpected role {text['from']} in {sample.texts}" + assert text["from"] in ["human", "gpt"], error_msg + conversation.append({ + "role": "user" if text["from"] == "human" else "assistant", + "content": text["value"]}) + + # Replace the image tags with IMAGE_TOKEN and count the number of image tags + number_image_tags = 0 + image_tag_ids_list = [] + for turn in conversation: + if turn["role"] == "user": + image_tag_ids = [int(x) - 1 for x in re.findall(r"", turn["content"])] + image_tag_ids_list.extend(image_tag_ids) + turn["content"] = re.sub(r"", IMAGE_TOKEN, turn["content"]) + number_image_tags += turn["content"].count(IMAGE_TOKEN) + # For videos, we replace the image tag with the video tag + if has_video: + turn["content"] = turn["content"].replace(IMAGE_TOKEN, VIDEO_TOKEN) + + # We re-order the images in sample.images according to how they appear in the conversation. + if len(image_tag_ids_list) > 0: + sample.images = [sample.images[idx] for idx in image_tag_ids_list] + + # If there is only one image, but several image tags, we assume all the tags refer to the + # same image and duplicate the image: + if len(sample.images) == 1 and number_image_tags > 1: + sample.images = sample.images * number_image_tags + + number_of_images = len(sample.images) + # Fail if there are more image or video tags than image or videos: + error_msg = ( + f"Found {number_image_tags} image tags for {number_of_images} images. {sample.texts}") + assert number_image_tags <= number_of_images, error_msg + + # If there are less image of video tags than image or videos, prepend the tags to the first + # user message: + if number_image_tags < number_of_images: for turn in conversation: if turn["role"] == "user": - turn["content"] = f"{IMAGE_TOKEN}\n" + turn["content"] + tag_to_add = VIDEO_TOKEN if has_video else IMAGE_TOKEN + turn["content"] = tag_to_add*(number_of_images-number_image_tags) + "\n" + turn["content"] break input_ids, target = self.tokenizer.tokenize_conversation(conversation, True, False) + if has_image: + imgs = [] + num_tiles = [] + max_num_tiles = self.args.max_num_tiles + # We keep a buffer of 4 tokens for the question, + # the rest can be used for image tokens. + max_image_token_allowed = self.args.decoder_seq_length - len(input_ids) - 4 + # We start by extracting as many tiles per image as possible, and decrease the max + # number of tiles if there are too many image tokens. + while True: + imgs = [] + num_tiles = [] + for img in sample.images: + img_tiles = get_visual_transform( + img, self.img_h, self.img_w, self.args.use_tiling, max_num_tiles, + self.args.use_thumbnail, augment, self.args.vision_model_type) + imgs += img_tiles + num_tiles += [len(img_tiles)] + if max_num_tiles == 1: + break + if sum(num_tiles) * self.token_per_img_tile > max_image_token_allowed: + if max_num_tiles in self.num_tiles_degradation_map: + max_num_tiles = self.num_tiles_degradation_map[max_num_tiles] + else: + raise RuntimeError(( + f"Tried to decrease the number of tiles {max_num_tiles} but it's not ", + f"defined in the degradation map {self.num_tiles_degradation_map}")) + else: + break + elif has_video: + # We don't use tiling for videos to limit the number of tokens. + use_tiling=False + # Grab the selected frames of the video as a tensor with shape + # fhwc: (num_frames, num_channels, height, width). + video_fchw = sample.images[0].permute(0, 1, 2, 3) + selected_frames = torch.linspace( + 0, video_fchw.shape[0] - 1, self.args.num_frames).long() + video_fchw = video_fchw[selected_frames] + imgs = [] + for video_chw in video_fchw: + to_pil = ToPILImage() + video_chw = to_pil(video_chw) + imgs += get_visual_transform( + video_chw, self.img_h, self.img_w, use_tiling, self.args.max_num_tiles, + self.args.use_thumbnail, augment, self.args.vision_model_type) + num_tiles = [len(imgs)] + else: + imgs = num_tiles = [] + if self.is_packing_enabled: input_ids, target = self._truncate_for_packing(input_ids, target, num_tiles) + # Some final checks with respect to the number of image tokens and images on the tokenized + # conversation. There can still be errors, for instance if a non-video sample happens to + # have our pre-defined video token, or if the packing truncation removed a necessary image + # tag. + number_image_token = np.sum(input_ids == self.img_token_id) + error_msg = ( + f"Found {number_image_token} image tokens for len({num_tiles}) = {len(num_tiles)} image tiles in {conversation}.") + assert number_image_token == len(num_tiles), error_msg + error_msg = ( + f"Found sum({num_tiles}) = {np.sum(num_tiles)} tiles for {len(imgs)} images in {conversation}.") + assert np.sum(num_tiles) == len(imgs), error_msg + return ImageTaskSample( __key__=sample.__key__, __restore_key__=sample.__restore_key__, @@ -407,8 +478,8 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, if isinstance(sample, MultiChoiceVQASample): cur_prompt = format_multichoice_question(sample.context, sample.choices) - if "" not in cur_prompt: - cur_prompt = "\n" + cur_prompt + if IMAGE_TOKEN not in cur_prompt: + cur_prompt = IMAGE_TOKEN + "\n" + cur_prompt cur_answer = format_multichoice_answer(sample.correct_choice_idx) elif isinstance(sample, VQASample): if 'docvqa' in sample.__key__: @@ -423,8 +494,8 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, cur_prompt = cur_prompt.format(sample.context) - if "" not in cur_prompt: - cur_prompt = "\n" + cur_prompt + if IMAGE_TOKEN not in cur_prompt: + cur_prompt = IMAGE_TOKEN + "\n" + cur_prompt if isinstance(sample.answers, list): answer_list = sample.answers @@ -505,11 +576,11 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, prompt_list = self.manual_prompts["DocPretraining"]["raw"] prompt_idx = np.random.randint(len(prompt_list)) cur_prompt = prompt_list[prompt_idx] - if "" not in cur_prompt: - cur_prompt = "\n" + cur_prompt + if IMAGE_TOKEN not in cur_prompt: + cur_prompt = IMAGE_TOKEN + "\n" + cur_prompt - # Make sure there is no extra tag. - sample.text = sample.text.replace("", "") + # Make sure there is no extra IMAGE_TOKEN tag. + sample.text = sample.text.replace(IMAGE_TOKEN, "") caption = sample.text.strip() @@ -526,8 +597,8 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, ref = sample.text region = sample.words_boxes - # Make sure there is no extra tag - ref = ref.replace("", "") + # Make sure there is no extra IMAGE_TOKEN tag + ref = ref.replace(IMAGE_TOKEN, "") if len(region) == 4: region = f"({region[0]},{region[1]}),({region[2]},{region[3]})" @@ -550,8 +621,8 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, prompt_idx = np.random.randint(len(prompt_list)) cur_prompt = prompt_list[prompt_idx] cur_prompt = cur_prompt.format(prompt_content) - if "" not in cur_prompt: - cur_prompt = "\n" + cur_prompt + if IMAGE_TOKEN not in cur_prompt: + cur_prompt = IMAGE_TOKEN + "\n" + cur_prompt return sample, cur_prompt, answer @@ -559,8 +630,8 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, """Format bbox coordinates as text.""" assert len(bbox) == 4 or len(bbox) == 8 - # Make sure there is no extra tag - text = text.replace("", "") + # Make sure there is no extra IMAGE_TOKEN tag + text = text.replace(IMAGE_TOKEN, "") if len(bbox) == 4: label_str = f"{text}({bbox[0]},{bbox[1]}),({bbox[2]},{bbox[3]})" @@ -582,8 +653,8 @@ class TaskEncoder(DefaultTaskEncoder[OCRSample, OCRSample, ImageTaskBatchPacked, prompt_idx = np.random.randint(len(prompt_list)) cur_prompt = prompt_list[prompt_idx] - if "" not in cur_prompt: - cur_prompt = "\n" + cur_prompt + if IMAGE_TOKEN not in cur_prompt: + cur_prompt = IMAGE_TOKEN + "\n" + cur_prompt cur_answer = answer return sample, cur_prompt, cur_answer diff --git a/examples/multimodal/evaluate_ai2d.py b/examples/multimodal/evaluation/evaluate_ai2d.py old mode 100755 new mode 100644 similarity index 72% rename from examples/multimodal/evaluate_ai2d.py rename to examples/multimodal/evaluation/evaluate_ai2d.py index 2d5db67..39b866a --- a/examples/multimodal/evaluate_ai2d.py +++ b/examples/multimodal/evaluation/evaluate_ai2d.py @@ -9,19 +9,25 @@ def merge_input_files(input_path): """Merge input files to a format compatible with the evaluator.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="AI2D") - results = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) - results.append( - { - "question_id": res["sample_id"], - "answer": res["answer"], - "gt_answer": res["gt_answer"], - } - ) + sample_id = res["sample_id"] + + # Ignore possible duplicates. + if sample_id in results: + continue + + results[sample_id] = { + "question_id": sample_id, + "answer": res["answer"], + "gt_answer": res["gt_answer"], + } + + results = list(results.values()) with open(output_file_path, "w") as output_file: json.dump(results, output_file) diff --git a/examples/multimodal/evaluate_chartqa.py b/examples/multimodal/evaluation/evaluate_chartqa.py old mode 100755 new mode 100644 similarity index 77% rename from examples/multimodal/evaluate_chartqa.py rename to examples/multimodal/evaluation/evaluate_chartqa.py index e923806..53d4944 --- a/examples/multimodal/evaluate_chartqa.py +++ b/examples/multimodal/evaluation/evaluate_chartqa.py @@ -9,15 +9,22 @@ def merge_input_files(input_path): """Merge input files to a format compatible with the evaluator.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="ChartQA") - results = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) - res["question_id"] = res["sample_id"] + sample_id = res["sample_id"] - results.append(res) + # Ignore possible duplicates. + if sample_id in results: + continue + + res["question_id"] = sample_id + results[sample_id] = res + + results = list(results.values()) with open(output_file_path, "w") as output_file: json.dump(results, output_file) diff --git a/examples/multimodal/evaluate_coco.py b/examples/multimodal/evaluation/evaluate_coco.py old mode 100755 new mode 100644 similarity index 77% rename from examples/multimodal/evaluate_coco.py rename to examples/multimodal/evaluation/evaluate_coco.py index a717090..8eeb367 --- a/examples/multimodal/evaluate_coco.py +++ b/examples/multimodal/evaluation/evaluate_coco.py @@ -11,20 +11,28 @@ def convert_to_coco_format(input_path): """Convert input files to COCO compatible format.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="captioning") - captions = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) + sample_id = res["sample_id"] - question_id = res['sample_id'] - caption = res['caption'].rstrip('.').lower() + # Ignore possible duplicates. + if sample_id in results: + continue - captions.append({"image_id": question_id, "caption": caption}) + caption = res["caption"].rstrip(".").lower() + results[sample_id] = { + "image_id": sample_id, + "caption": caption, + } + + results = list(results.values()) with open(output_file_path, "w") as output_file: - json.dump(captions, output_file, indent=4) + json.dump(results, output_file, indent=4) return output_file_path diff --git a/examples/multimodal/evaluate_mathvista.py b/examples/multimodal/evaluation/evaluate_mathvista.py old mode 100755 new mode 100644 similarity index 92% rename from examples/multimodal/evaluate_mathvista.py rename to examples/multimodal/evaluation/evaluate_mathvista.py index 3474c5f..a55f312 --- a/examples/multimodal/evaluate_mathvista.py +++ b/examples/multimodal/evaluation/evaluate_mathvista.py @@ -11,13 +11,21 @@ def merge_input_files(input_path): """Merge input files to a format compatible with the evaluator.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="MathVista") - results = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) - results.append(res) + sample_id = res["sample_id"] + + # Remove possible duplicates. + if sample_id in results: + continue + + results[sample_id] = res + + results = list(results.values()) with open(output_file_path, "w") as output_file: json.dump(results, output_file) diff --git a/examples/multimodal/evaluate_mmmu.py b/examples/multimodal/evaluation/evaluate_mmmu.py old mode 100755 new mode 100644 similarity index 91% rename from examples/multimodal/evaluate_mmmu.py rename to examples/multimodal/evaluation/evaluate_mmmu.py index 66118fa..798c42b --- a/examples/multimodal/evaluate_mmmu.py +++ b/examples/multimodal/evaluation/evaluate_mmmu.py @@ -2,9 +2,15 @@ import argparse import glob import json import os +import sys import re import subprocess +# Get the absolute path of the parent directory +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) +# Add the parent directory to sys.path +sys.path.insert(0, parent_dir) + from run_text_generation import get_output_path from config import EvaluationConfig @@ -48,6 +54,10 @@ def convert_to_mmmu_format(input_path): ) # MMMU eval script expects just a sample_id to prediction mapping. + # Skip possible duplicates. + if sample_id in output: + continue + output[sample_id] = prediction with open(output_file_path, "w") as output_file: diff --git a/examples/multimodal/evaluate_ocrbench.py b/examples/multimodal/evaluation/evaluate_ocrbench.py old mode 100755 new mode 100644 similarity index 95% rename from examples/multimodal/evaluate_ocrbench.py rename to examples/multimodal/evaluation/evaluate_ocrbench.py index bc2b901..b37473a --- a/examples/multimodal/evaluate_ocrbench.py +++ b/examples/multimodal/evaluation/evaluate_ocrbench.py @@ -8,13 +8,21 @@ def merge_input_files(input_path): """Merge input files to a format compatible with the evaluator.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="OCRBench") - results = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) - results.append(res) + sample_id = res["sample_id"] + + # Remove possible duplicates. + if sample_id in results: + continue + + results[sample_id] = res + + results = list(results.values()) with open(output_file_path, "w") as output_file: json.dump(results, output_file) diff --git a/examples/multimodal/evaluate_textvqa.py b/examples/multimodal/evaluation/evaluate_textvqa.py old mode 100755 new mode 100644 similarity index 72% rename from examples/multimodal/evaluate_textvqa.py rename to examples/multimodal/evaluation/evaluate_textvqa.py index c9bba71..af782bd --- a/examples/multimodal/evaluate_textvqa.py +++ b/examples/multimodal/evaluation/evaluate_textvqa.py @@ -9,22 +9,25 @@ def merge_input_files(input_path): """Merge input files to a format compatible with the evaluator.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="TextVQA") - results = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) - results.append( - { - "question_id": res["sample_id"], - "answer": res["answer"], - "gt_answer": res["gt_answer"], - } - ) - - # Make order deterministic. - # results = sorted(results, key=lambda d: d["question_id"]) + sample_id = res["sample_id"] + + # Remove possible duplicates. + if sample_id in results: + continue + + results[sample_id] = { + "question_id": sample_id, + "answer": res["answer"], + "gt_answer": res["gt_answer"], + } + + results = list(results.values()) with open(output_file_path, "w") as output_file: json.dump(results, output_file) diff --git a/examples/multimodal/evaluate_vqav2.py b/examples/multimodal/evaluation/evaluate_vqav2.py old mode 100755 new mode 100644 similarity index 88% rename from examples/multimodal/evaluate_vqav2.py rename to examples/multimodal/evaluation/evaluate_vqav2.py index 0b1b920..7807d80 --- a/examples/multimodal/evaluate_vqav2.py +++ b/examples/multimodal/evaluation/evaluate_vqav2.py @@ -9,15 +9,22 @@ def merge_input_files(input_path): """Merge input files to a format compatible with the evaluator.""" input_file_paths, output_file_path = get_input_output_paths(input_path, task="VQAv2") - results = [] + results = dict() for input_file_path in input_file_paths: with open(input_file_path, "r") as input_file: for line in input_file: res = json.loads(line) - res["question_id"] = res["sample_id"] + sample_id = res["sample_id"] - results.append(res) + # Skip possible duplicates. + if sample_id in results: + continue + + res["question_id"] = sample_id + results[sample_id] = res + + results = list(results.values()) with open(output_file_path, "w") as output_file: json.dump(results, output_file) @@ -57,6 +64,9 @@ def compute_vqa_accuracy(result_file, task): assert len(gt) == 1, "expected exactly one groundtruth answer." gt = gt[0] + pred = pred.rstrip("%") + gt = gt.rstrip("%") + if is_number(pred) and is_number(gt): pred = float(pred) gt = float(gt) diff --git a/examples/multimodal/evaluation_datasets.py b/examples/multimodal/evaluation/evaluation_datasets.py old mode 100755 new mode 100644 similarity index 88% rename from examples/multimodal/evaluation_datasets.py rename to examples/multimodal/evaluation/evaluation_datasets.py index 97f9ba9..50a50d5 --- a/examples/multimodal/evaluation_datasets.py +++ b/examples/multimodal/evaluation/evaluation_datasets.py @@ -188,7 +188,7 @@ class MMMUDataset(torch.utils.data.Dataset): use_tiling, max_num_tiles, use_thumbnail, - single_image, + prompt_style, vision_model_type, ): import datasets @@ -246,7 +246,7 @@ class MMMUDataset(torch.utils.data.Dataset): self._use_tiling = use_tiling self._max_num_tiles = max_num_tiles self._use_thumbnail = use_thumbnail - self._single_image = single_image + self._prompt_style = prompt_style self._vision_model_type = vision_model_type def __len__(self): @@ -258,7 +258,7 @@ class MMMUDataset(torch.utils.data.Dataset): sample = self._dataset[idx] # Use the single image approach from the MMMU repo. - if self._single_image: + if self._prompt_style == "single_image": sample = process_single_sample(sample) sample = construct_prompt(sample, self._config) @@ -274,7 +274,69 @@ class MMMUDataset(torch.utils.data.Dataset): vision_model_type=self._vision_model_type, ) sample_num_tiles = [len(sample_imgs)] - else: + + prompt = sample["final_input_prompt"] + for i in range(8): + prompt = prompt.replace(f"", "") + sample["final_input_prompt"] = f"\n{prompt}" + elif self._prompt_style == "vlmevalkit": + sample = construct_prompt(sample, self._config) + + if sample["question_type"] == "multiple-choice": + question = sample["question"] + + options = "" + for k, v in sample["index2ans"].items(): + options += f"{k}. {v}\n" + + final_prompt = f"{question}\n" + if "hint" in sample: + final_prompt += f"Hint: {sample['hint']}\n" + + if "task_instructions" in sample: + final_prompt += f"Task instructions: {sample['task_instructions']}\n" + + final_prompt += options + final_prompt += "Answer with the option's letter from the given choices directly." + + sample["final_input_prompt"] = final_prompt.rstrip() + else: + question = sample["question"] + final_prompt = f"{question}\n" + final_prompt += "Answer the question directly." + sample["final_input_prompt"] = final_prompt.rstrip() + + sample_imgs = [] + sample_num_tiles = [] + + img_indices = sorted(list(set(re.findall(r"" + + img = sample[img_key] + assert img is not None, f"{img_str} is in prompt but not in sample images" + + imgs = get_visual_transform( + img, + self._img_h, + self._img_w, + self._use_tiling, + adjusted_max_num_tiles, + self._use_thumbnail, + augment=False, + vision_model_type=self._vision_model_type, + ) # List of tiles. + + sample_imgs.extend(imgs) + sample_num_tiles.append(len(imgs)) + + sample["final_input_prompt"] = " ".join([f'' for i in range(len(img_indices))]) + "\n" + sample["final_input_prompt"] + elif self._prompt_style == "multi_image": sample = construct_prompt(sample, self._config) sample_imgs = [] @@ -315,6 +377,8 @@ class MMMUDataset(torch.utils.data.Dataset): assert ( f"" not in sample["final_input_prompt"] ), "prompt contains unhandled image tags" + else: + raise ValueError(f"unknown prompt style {self._prompt_style}") # MMMU specific metadata. metadata = {"question_type": sample["question_type"]} @@ -323,10 +387,6 @@ class MMMUDataset(torch.utils.data.Dataset): metadata["all_choices"] = sample["all_choices"] prompt = sample['final_input_prompt'] - if self._single_image: - for i in range(8): - prompt = prompt.replace(f"", "") - prompt = f"\n{prompt}" tile_count = torch.tensor(sample_num_tiles, dtype=torch.int) @@ -780,8 +840,10 @@ def get_evaluation_dataset( vision_model_type, ) elif task == 'MMMU': - # Note: single_image=True uses only one image like in the MMMU repo example. - # single_image=False uses all images in the sample. + # Note: + # - prompt_style="single_image" uses only one image like in the MMMU repo example. + # - prompt_style="multi_image" uses multiple input images. + # - prompt_style="vlmevalkit" is similar to https://github.com/open-compass/VLMEvalKit/blob/5d3cebcf18ef4bfbadc3bd3ef80bdc7aad2c6557/vlmeval/vlm/internvl_chat.py#L499 dataset = MMMUDataset( input_image_path, num_samples_per_partition, @@ -792,7 +854,7 @@ def get_evaluation_dataset( use_tiling, max_num_tiles, use_thumbnail, - single_image=True, + prompt_style="single_image", vision_model_type=vision_model_type, ) elif task == "VideoMME": diff --git a/examples/multimodal/image_processing.py b/examples/multimodal/image_processing.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/layer_specs.py b/examples/multimodal/layer_specs.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/manual_prompts.json b/examples/multimodal/manual_prompts.json old mode 100755 new mode 100644 diff --git a/examples/multimodal/model.py b/examples/multimodal/model.py old mode 100755 new mode 100644 index 6db834e..a28a428 --- a/examples/multimodal/model.py +++ b/examples/multimodal/model.py @@ -136,6 +136,20 @@ def model_provider( else: vision_projection_layer_spec = get_mlp_module_spec(use_te=use_te).submodules + # Toggle --recompute* for the vision and language model separately. + if args.recompute_vision: + if vision_config.recompute_method is not None and vision_config.recompute_granularity is not None: + vision_config.recompute_num_layers = vision_config.num_layers + else: + vision_config.recompute_granularity = None + vision_config.recompute_method = None + vision_config.recompute_num_layers = None + + vision_projection_config.recompute_granularity = None + vision_projection_config.recompute_method = None + vision_projection_config.recompute_num_layers = None + + tokenizer = get_tokenizer() image_token_index = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) diff --git a/examples/multimodal/model_converter/clip_converter.py b/examples/multimodal/model_converter/clip_converter.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/model_converter/internvit_converter.py b/examples/multimodal/model_converter/internvit_converter.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/model_converter/siglip_converter.py b/examples/multimodal/model_converter/siglip_converter.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/model_converter/vision_model_tester.py b/examples/multimodal/model_converter/vision_model_tester.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/multimodal_args.py b/examples/multimodal/multimodal_args.py old mode 100755 new mode 100644 index 4b2be45..eb56118 --- a/examples/multimodal/multimodal_args.py +++ b/examples/multimodal/multimodal_args.py @@ -49,7 +49,7 @@ def add_multimodal_extra_args(parser): group.add_argument( "--tokenizer-prompt-format", type=str, - choices=["mistral", "llama3", "chatml", "nvlm-yi-34b", "qwen2p0"], + choices=["mistral", "llama3", "chatml", "nvlm-yi-34b", "qwen2p0", "qwen2p5"], required=True, help="Prompt format to use with the tokenizer.", ) @@ -71,5 +71,9 @@ def add_multimodal_extra_args(parser): group.add_argument( "--packing-seq-length", type=int, default=0, help="Packing sequence length. Must be > 0 if using packing." ) + group.add_argument( + "--recompute-vision", action="store_true", default=False, help="Enable activation checkpointing in the vision model" + ) + return parser diff --git a/examples/multimodal/nvlm/README.md b/examples/multimodal/nvlm/README.md old mode 100755 new mode 100644 index 7eddbb7..bb576bb --- a/examples/multimodal/nvlm/README.md +++ b/examples/multimodal/nvlm/README.md @@ -5,6 +5,13 @@ Please refer to the [NVLM paper](https://arxiv.org/pdf/2409.11402) for details. *NOTE: VLMs in Megatron are under active development and are expected to change.* +# Checkpoints + +NVLM 1.0 model weights are publicly available in HuggingFace and Megatron format. + +- NVLM-1.0-D 72B [HuggingFace version](https://huggingface.co/nvidia/NVLM-D-72B) +- NVLM-1.0-D 72B [Megatron-Core version](https://huggingface.co/nvidia/NVLM-D-72B-mcore) + # Setup ## Docker image @@ -32,7 +39,7 @@ NVLM 1.0 34B starts from [NousResearch/Nous-Hermes-2-Yi-34B](https://huggingface Please download it and run the following command to convert it to Megatron format. ``` python tools/checkpoint/convert.py --bf16 --model-type GPT --loader llama_mistral --saver mcore --target-tensor-parallel-size 8 --checkpoint-type hf \ - --load-dir --save-dir --tokenizer-model \ + --load-dir --save-dir --tokenizer-model \ --saver-transformer-impl transformer_engine --model-size yi-34B --make-vocab-size-divisible-by 1 ``` @@ -42,7 +49,7 @@ NVLM 1.0 72B starts from [Qwen/Qwen2-72B-Instruct](https://huggingface.co/Qwen/Q Please download it and run the following command to convert it to Megatron format. ``` python tools/checkpoint/convert.py --bf16 --model-type GPT --loader llama_mistral --saver mcore --target-tensor-parallel-size 8 --checkpoint-type hf \ - --load-dir --save-dir --tokenizer-model \ + --load-dir --save-dir --tokenizer-model \ --saver-transformer-impl transformer_engine --model-size qwen2.5-72Bf ``` diff --git a/examples/multimodal/nvlm/internvit.py b/examples/multimodal/nvlm/internvit.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/nvlm/nvlm_prompts.json b/examples/multimodal/nvlm/nvlm_prompts.json old mode 100755 new mode 100644 diff --git a/examples/multimodal/nvlm/pp_checkpoint_converter.py b/examples/multimodal/nvlm/pp_checkpoint_converter.py old mode 100755 new mode 100644 diff --git a/examples/multimodal/nvlm/pretrain_blend.yaml b/examples/multimodal/nvlm/pretrain_blend.yaml old mode 100755 new mode 100644 diff --git a/examples/multimodal/nvlm/pretrain_qwen20_72b_internvit_6b.sh b/examples/multimodal/nvlm/pretrain_qwen20_72b_internvit_6b.sh old mode 100755 new mode 100644 index 320c7ad..008a17a --- a/examples/multimodal/nvlm/pretrain_qwen20_72b_internvit_6b.sh +++ b/examples/multimodal/nvlm/pretrain_qwen20_72b_internvit_6b.sh @@ -62,7 +62,7 @@ OPTIONS=" \ --exit-duration-in-mins 230 \ --disable-bias-linear \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${WORKSPACE}/ \ + --tokenizer-model Qwen/Qwen2-72B-Instruct \ --tokenizer-prompt-format qwen2p0 \ --transformer-impl transformer_engine \ --normalization RMSNorm \ diff --git a/examples/multimodal/nvlm/pretrain_yi_34b_internvit_6b.sh b/examples/multimodal/nvlm/pretrain_yi_34b_internvit_6b.sh old mode 100755 new mode 100644 index c36cb05..00f9435 --- a/examples/multimodal/nvlm/pretrain_yi_34b_internvit_6b.sh +++ b/examples/multimodal/nvlm/pretrain_yi_34b_internvit_6b.sh @@ -75,7 +75,7 @@ OPTIONS=" \ --decoder-seq-length ${DECODER_SEQ_LEN} \ --max-position-embeddings ${MAX_POS_EMBED} \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${WORKSPACE}/ \ + --tokenizer-model NousResearch/Nous-Hermes-2-Yi-34B \ --tokenizer-prompt-format nvlm-yi-34b \ --vocab-size 64000 \ --make-vocab-size-divisible-by 1 \ diff --git a/examples/multimodal/nvlm/run_text_generation_qwen20_72b_internvit_6b.sh b/examples/multimodal/nvlm/run_text_generation_qwen20_72b_internvit_6b.sh old mode 100755 new mode 100644 index 35cd904..e3b001c --- a/examples/multimodal/nvlm/run_text_generation_qwen20_72b_internvit_6b.sh +++ b/examples/multimodal/nvlm/run_text_generation_qwen20_72b_internvit_6b.sh @@ -97,7 +97,7 @@ do --decoder-seq-length ${DECODER_SEQ_LEN} \ --max-position-embeddings ${MAX_POS_EMBED} \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model \ + --tokenizer-model Qwen/Qwen2-72B-Instruct \ --tokenizer-prompt-format qwen2p0 \ --position-embedding-type rope \ --rotary-percent 1.0 \ diff --git a/examples/multimodal/nvlm/run_text_generation_qwen25_7b_siglip.sh b/examples/multimodal/nvlm/run_text_generation_qwen25_7b_siglip.sh new file mode 100644 index 0000000..3b62219 --- /dev/null +++ b/examples/multimodal/nvlm/run_text_generation_qwen25_7b_siglip.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +export NCCL_IB_SL=1 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NVTE_APPLY_QK_LAYER_SCALING=0 +export TOKENIZERS_PARALLELISM="false" + +INPUT_IMAGE_PATH="placeholder" +GROUNDTRUTH_PATH="placeholder" + +while [[ $# -gt 0 ]]; do + case $1 in + -i|--input-image-path) + INPUT_IMAGE_PATH="$2" + shift + shift + ;; + -o|--output-path) + OUTPUT_PATH="$2" + shift + shift + ;; + -m|--model-path) + MODEL_PATH="$2" + shift + shift + ;; + -t|--task) + TASK="$2" + shift + shift + ;; + -g|--gt-path) + GROUNDTRUTH_PATH="$2" + shift + shift + ;; + -*|--*) + echo "Invalid option $1" + exit 1 + ;; + esac +done + +# Please modify these as needed. +NUM_PARTITIONS=0 +START=0 +END=0 + + +SEQ_LEN=256 +DECODER_SEQ_LEN=8192 +EXTRA_ARGS=" --pixel-shuffle --use-tiling --max-num-tiles 12 --use-thumbnail" + +for PARTITION_ID in $( eval echo {$START..$END} ) +do + torchrun --nproc_per_node 8 examples/multimodal/run_text_generation.py \ + --attention-softmax-in-fp32 \ + --transformer-impl transformer_engine \ + --use-te \ + --use-checkpoint-args \ + --normalization RMSNorm \ + --norm-epsilon 1e-06 \ + --language-model-type=qwen2.5_7B \ + --untie-embeddings-and-output-weights \ + --disable-bias-linear \ + --position-embedding-type rope \ + --rotary-percent 1.0 \ + --rotary-base 1000000 \ + --swiglu \ + --attention-dropout 0.0 \ + --hidden-dropout 0.0 \ + --tensor-model-parallel-size 4 \ + --pipeline-model-parallel-size 1 \ + --group-query-attention \ + --num-query-groups 4 \ + --num-layers 28 \ + --hidden-size 3584 \ + --ffn-hidden-size 18944 \ + --add-qkv-bias \ + --num-attention-heads 28 \ + --max-position-embeddings 32768 \ + --no-masked-softmax-fusion \ + --load ${MODEL_PATH} \ + --tokenizer-type MultimodalTokenizer \ + --tokenizer-model Qwen/Qwen2.5-7B-Instruct \ + --tokenizer-prompt-format qwen2p5 \ + --bf16 \ + --micro-batch-size 1 \ + --seq-length ${SEQ_LEN} \ + --decoder-seq-length ${DECODER_SEQ_LEN} \ + --out-seq-length 128 \ + --temperature 1.0 \ + --img-h 448 \ + --img-w 448 \ + --patch-dim 14 \ + --seed 153 \ + --top_k 1 \ + --no-load-rng \ + --no-load-optim \ + --input-image-path ${INPUT_IMAGE_PATH} \ + --num-partitions ${NUM_PARTITIONS} \ + --partition-id ${PARTITION_ID} \ + --output-path ${OUTPUT_PATH} \ + --gt-path ${GROUNDTRUTH_PATH} \ + --task ${TASK} \ + ${EXTRA_ARGS} \ + --special-tokens "" "" "" \ + --vision-model-type siglip \ + --ckpt-format torch +done diff --git a/examples/multimodal/nvlm/run_text_generation_yi_34b_internvit_6b.sh b/examples/multimodal/nvlm/run_text_generation_yi_34b_internvit_6b.sh old mode 100755 new mode 100644 index 0437e4c..341f4e4 --- a/examples/multimodal/nvlm/run_text_generation_yi_34b_internvit_6b.sh +++ b/examples/multimodal/nvlm/run_text_generation_yi_34b_internvit_6b.sh @@ -95,7 +95,7 @@ do --decoder-seq-length ${DECODER_SEQ_LEN} \ --max-position-embeddings ${MAX_POS_EMBED} \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model \ + --tokenizer-model NousResearch/Nous-Hermes-2-Yi-34B \ --tokenizer-prompt-format nvlm-yi-34b \ --vocab-size 64000 \ --make-vocab-size-divisible-by 1 \ @@ -135,6 +135,6 @@ do --gt-path ${GROUNDTRUTH_PATH} \ ${EXTRA_ARGS} \ --task ${TASK} \ - --image-tag-type nlvm \ + --image-tag-type nvlm \ --ckpt-format torch done diff --git a/examples/multimodal/nvlm/sft_34b_internvit.sh b/examples/multimodal/nvlm/sft_34b_internvit.sh old mode 100755 new mode 100644 index 3d585d8..0dff946 --- a/examples/multimodal/nvlm/sft_34b_internvit.sh +++ b/examples/multimodal/nvlm/sft_34b_internvit.sh @@ -80,7 +80,7 @@ OPTIONS=" \ --decoder-seq-length ${DECODER_SEQ_LEN} \ --max-position-embeddings ${MAX_POS_EMBED} \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${WORKSPACE}/ \ + --tokenizer-model NousResearch/Nous-Hermes-2-Yi-34B \ --tokenizer-prompt-format nvlm-yi-34b \ --vocab-size 64000 \ --make-vocab-size-divisible-by 1 \ diff --git a/examples/multimodal/nvlm/sft_blend.yaml b/examples/multimodal/nvlm/sft_blend.yaml old mode 100755 new mode 100644 diff --git a/examples/multimodal/nvlm/sft_qwen20_72b_internvit_6b.sh b/examples/multimodal/nvlm/sft_qwen20_72b_internvit_6b.sh old mode 100755 new mode 100644 index adb1d1b..3b47225 --- a/examples/multimodal/nvlm/sft_qwen20_72b_internvit_6b.sh +++ b/examples/multimodal/nvlm/sft_qwen20_72b_internvit_6b.sh @@ -67,7 +67,7 @@ OPTIONS=" \ --exit-duration-in-mins 230 \ --disable-bias-linear \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${WORKSPACE}/ \ + --tokenizer-model Qwen/Qwen2-72B-Instruct \ --tokenizer-prompt-format qwen2p0 \ --transformer-impl transformer_engine \ --normalization RMSNorm \ diff --git a/examples/multimodal/pretrain_dataset.yaml b/examples/multimodal/pretrain_dataset.yaml old mode 100755 new mode 100644 diff --git a/examples/multimodal/pretrain_mistral_clip.sh b/examples/multimodal/pretrain_mistral_clip.sh old mode 100755 new mode 100644 index ea1f741..90b0053 --- a/examples/multimodal/pretrain_mistral_clip.sh +++ b/examples/multimodal/pretrain_mistral_clip.sh @@ -24,11 +24,6 @@ if [[ -z $LOAD_NAME ]]; then exit 1 fi -if [[ -z $TOKENIZER_MODEL ]]; then - echo "Please set TOKENIZER_MODEL for tokenizer model name." - exit 1 -fi - CHECKPOINT_DIR="${WORKSPACE}/${LOAD_NAME}/checkpoints" DATA_TRAIN="${SOURCE}/examples/multimodal/pretrain_dataset.yaml" @@ -93,7 +88,7 @@ OPTIONS=" \ --eval-iters 10 \ --eval-interval 1000 \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${WORKSPACE}/${TOKENIZER_MODEL} \ + --tokenizer-model mistralai/Mistral-7B-Instruct-v0.3 \ --tokenizer-prompt-format mistral \ --data-path ${DATA_TRAIN} \ --prompt-path ${SOURCE}/examples/multimodal/manual_prompts.json \ diff --git a/examples/multimodal/run_text_generation.py b/examples/multimodal/run_text_generation.py old mode 100755 new mode 100644 index f4bb502..cbde668 --- a/examples/multimodal/run_text_generation.py +++ b/examples/multimodal/run_text_generation.py @@ -14,11 +14,13 @@ sys.path.append( import torch import yaml from config import EvaluationConfig -from evaluation_datasets import get_evaluation_dataset +from evaluation.evaluation_datasets import get_evaluation_dataset from model import model_provider from multimodal_args import add_multimodal_extra_args from megatron.core import parallel_state +from megatron.core.enums import ModelType +from megatron.core.models.multimodal.llava_model import IMAGE_TOKEN from megatron.core.models.vision.clip_vit_model import get_num_image_embeddings from megatron.inference.text_generation.api import generate_and_post_process from megatron.inference.text_generation.forward_step import ForwardStep @@ -36,7 +38,7 @@ def add_text_generation_args(parser): group.add_argument("--top_p", type=float, default=0.0, help='Top p sampling.') group.add_argument("--top_k", type=int, default=0, help='Top k sampling.') group.add_argument( - "--out-seq-length", type=int, default=1024, help='Length of the output generated text.' + "--out-seq-length", type=int, default=128, help='Length of the output generated text.' ) group.add_argument("--output-path", type=str, help='Output file path') group.add_argument('--input-image-path', type=str, help="Input image directory") @@ -206,8 +208,8 @@ def generate_samples(model, config: EvaluationConfig, print_output): if config.task == "VideoMME": output["questions"][0][output_name] = generated else: - output[output_name] = generated output["prompt"] = prompt + output[output_name] = generated if config.task == "captioning": output["ground_truth"] = answers @@ -354,7 +356,7 @@ class VLMForwardStep(ForwardStep): ) def __call__(self, tokens, position_ids, attention_mask): - num_image_tokens = (tokens == self.model.image_token_index).sum().item() + num_image_tokens = (tokens == self.model.module.image_token_index).sum().item() num_tokens = tokens.size(1) recv_buffer_seq_length = None if num_image_tokens > 0: @@ -406,7 +408,7 @@ def get_conversation(task, question): {"role": "system", "content": "Answer the questions."}, { "role": "user", - "content": "\nProvide a one-sentence caption for provided image.", + "content": f"{IMAGE_TOKEN}\nProvide a one-sentence caption for provided image.", }, ] elif task in ("TextVQA", "VQAv2", "ChartQA"): @@ -414,13 +416,13 @@ def get_conversation(task, question): {"role": "system", "content": "Answer the questions."}, { "role": "user", - "content": f"\n{question}\nAnswer the question using a single word or phrase.", + "content": f"{IMAGE_TOKEN}\n{question}\nAnswer the question using a single word or phrase.", }, ] elif task in ("OCRBench", "MathVista", "AI2D"): conversation = [ {"role": "system", "content": "Answer the questions."}, - {"role": "user", "content": f"\n{question}"}, + {"role": "user", "content": f"{IMAGE_TOKEN}\n{question}"}, ] elif task == "MMMU": conversation = [ @@ -441,7 +443,7 @@ def get_conversation(task, question): conversation = [ {"role": "system", "content": "Answer the questions."}, - {"role": "user", "content": f"\n{question}"}, + {"role": "user", "content": f"{IMAGE_TOKEN}\n{question}"}, ] return conversation @@ -464,11 +466,13 @@ def get_prompt_and_generated(prompt_and_generation, prompt_format): prompt = splitted[0] generated = splitted[1] generated = generated.split("<|im_end|>")[0] - elif prompt_format in ("nvlm-yi-34b", "qwen2p0"): + elif prompt_format in ("nvlm-yi-34b", "qwen2p0", "qwen2p5"): splitted = prompt_and_generation.split("<|im_start|>assistant\n") prompt = splitted[0] generated = splitted[1] generated = generated.split("<|im_end|>")[0] + else: + raise ValueError(f"Prompt format {prompt_format} is not supported.") # Remove possible garbage. generated = generated.strip() @@ -489,11 +493,11 @@ def main(): args = get_args() - def wrapped_model_provider(pre_process, post_process): - return model_provider(pre_process, post_process, parallel_output=False) + def wrapped_model_provider(pre_process, post_process, add_encoder, add_decoder): + return model_provider(pre_process, post_process, add_encoder, add_decoder, parallel_output=False) # Set up model and load checkpoint. - model = get_model(wrapped_model_provider, wrap_with_ddp=False) + model = get_model(wrapped_model_provider, model_type=ModelType.encoder_and_decoder, wrap_with_ddp=False) if args.load is not None: _ = load_checkpoint(model, None, None) diff --git a/examples/multimodal/sft_dataset.yaml b/examples/multimodal/sft_dataset.yaml old mode 100755 new mode 100644 diff --git a/examples/multimodal/sft_mistral_clip.sh b/examples/multimodal/sft_mistral_clip.sh old mode 100755 new mode 100644 index 8a083cc..94ff208 --- a/examples/multimodal/sft_mistral_clip.sh +++ b/examples/multimodal/sft_mistral_clip.sh @@ -29,11 +29,6 @@ if [[ -z $LOAD_ITER ]]; then exit 1 fi -if [[ -z $TOKENIZER_MODEL ]]; then - echo "Please set TOKENIZER_MODEL for tokenizer model name." - exit 1 -fi - CHECKPOINT_DIR="${WORKSPACE}/${LOAD_NAME}/checkpoints" DATA_TRAIN="${SOURCE}/examples/multimodal/sft_dataset.yaml" @@ -98,7 +93,7 @@ OPTIONS=" \ --eval-iters 10 \ --eval-interval 500 \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${WORKSPACE}/${TOKENIZER_MODEL} \ + --tokenizer-model mistralai/Mistral-7B-Instruct-v0.3 \ --tokenizer-prompt-format mistral \ --data-path ${DATA_TRAIN} \ --prompt-path ${SOURCE}/examples/multimodal/manual_prompts.json \ diff --git a/examples/multimodal/text_generation_mistral_clip.sh b/examples/multimodal/text_generation_mistral_clip.sh old mode 100755 new mode 100644 index ca98ff2..c1ef7bc --- a/examples/multimodal/text_generation_mistral_clip.sh +++ b/examples/multimodal/text_generation_mistral_clip.sh @@ -4,12 +4,13 @@ export NCCL_IB_SL=1 export CUDA_DEVICE_MAX_CONNECTIONS=1 export NVTE_APPLY_QK_LAYER_SCALING=0 +INPUT_IMAGE_PATH="placeholder" GROUNDTRUTH_PATH="placeholder" NUM_FRAMES=1 while [[ $# -gt 0 ]]; do case $1 in - --input-image-path) + -i|--input-image-path) INPUT_IMAGE_PATH="$2" shift shift @@ -19,11 +20,6 @@ while [[ $# -gt 0 ]]; do shift shift ;; - -g|--groundtruth-path) - GROUNDTRUTH_PATH="$2" - shift - shift - ;; -o|--output-path) OUTPUT_PATH="$2" shift @@ -34,12 +30,7 @@ while [[ $# -gt 0 ]]; do shift shift ;; - -t|--tokenizer-path) - TOKENIZER_PATH="$2" - shift - shift - ;; - --task) + -t|--task) TASK="$2" shift shift @@ -92,7 +83,7 @@ do --no-masked-softmax-fusion \ --load ${MODEL_PATH} \ --tokenizer-type MultimodalTokenizer \ - --tokenizer-model ${TOKENIZER_PATH} \ + --tokenizer-model mistralai/Mistral-7B-Instruct-v0.3 \ --tokenizer-prompt-format mistral \ --bf16 \ --micro-batch-size 1 \ diff --git a/examples/multimodal/train.py b/examples/multimodal/train.py old mode 100755 new mode 100644 index 5ff2121..1dc68d1 --- a/examples/multimodal/train.py +++ b/examples/multimodal/train.py @@ -48,7 +48,7 @@ def get_batch(data_iterator): pp_size = get_pipeline_model_parallel_world_size() if not is_first_or_last_stage(pp_size, args.encoder_pipeline_model_parallel_size): # Note these are all set to None above. - return tokens, labels, loss_mask, attention_mask, position_ids, imgs, num_tiles + return tokens, labels, loss_mask, attention_mask, position_ids, imgs, num_tiles, packed_seq_params # Broadcast data. torch.cuda.nvtx.range_push("get_data") @@ -66,7 +66,7 @@ def get_batch(data_iterator): cu_lengths = tensor_parallel.broadcast_data(["cu_lengths"], data, torch.int32)["cu_lengths"] max_lengths = tensor_parallel.broadcast_data(["max_lengths"], data, torch.int32)["max_lengths"] - # Dummy image, no image. + # No image input (text-only sample) if the dataloader produced a dummy image. if imgs.shape == torch.Size([1, 1]): # FIXME: text-only data can cause a hang if the vision model is own its own pipeline rank and --freeze-ViT is enabled. imgs = torch.tensor([], dtype=torch.float32, device=data_text.device) diff --git a/examples/retro/README.md b/examples/retro/README.md old mode 100755 new mode 100644 diff --git a/examples/retro/preprocess_data.sh b/examples/retro/preprocess_data.sh old mode 100755 new mode 100644 diff --git a/examples/retro/train_retro_2b_distributed.sh b/examples/retro/train_retro_2b_distributed.sh old mode 100755 new mode 100644 diff --git a/examples/run_simple_mcore_train_loop.py b/examples/run_simple_mcore_train_loop.py old mode 100755 new mode 100644 diff --git a/examples/t5/README.md b/examples/t5/README.md old mode 100755 new mode 100644 diff --git a/examples/t5/t5_mcore_train_curve.png b/examples/t5/t5_mcore_train_curve.png old mode 100755 new mode 100644 diff --git a/examples/t5/train_t5_220m_distributed.sh b/examples/t5/train_t5_220m_distributed.sh old mode 100755 new mode 100644 diff --git a/hostfile_gptmoe b/hostfile_gptmoe new file mode 100644 index 0000000..7dd871e --- /dev/null +++ b/hostfile_gptmoe @@ -0,0 +1,16 @@ +node007 slots=8 +node008 slots=8 +node009 slots=8 +node010 slots=8 +node011 slots=8 +node012 slots=8 +node013 slots=8 +node014 slots=8 +node015 slots=8 +node016 slots=8 +node017 slots=8 +node018 slots=8 +node019 slots=8 +node020 slots=8 +node021 slots=8 +node022 slots=8 diff --git a/hostfile_mixtral8x7B b/hostfile_mixtral8x7B new file mode 100644 index 0000000..582f847 --- /dev/null +++ b/hostfile_mixtral8x7B @@ -0,0 +1,2 @@ +node007 slots=8 +node008 slots=8 diff --git a/images/model_table.png b/images/model_table.png old mode 100755 new mode 100644 diff --git a/images/strong_scaling.png b/images/strong_scaling.png old mode 100755 new mode 100644 diff --git a/images/weak_scaling.png b/images/weak_scaling.png old mode 100755 new mode 100644 diff --git a/kill.sh b/kill.sh new file mode 100644 index 0000000..688f93a --- /dev/null +++ b/kill.sh @@ -0,0 +1,3 @@ +ps -ef | grep python3 | grep -v grep | awk '{print "kill "$2}' >& .tmp +bash .tmp + diff --git a/megatron/core/QuickStart.md b/megatron/core/QuickStart.md old mode 100755 new mode 100644 diff --git a/megatron/core/README.md b/megatron/core/README.md old mode 100755 new mode 100644 diff --git a/megatron/core/README_STRAGGLER.md b/megatron/core/README_STRAGGLER.md old mode 100755 new mode 100644 diff --git a/megatron/core/__init__.py b/megatron/core/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/config_logger.py b/megatron/core/config_logger.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/Makefile b/megatron/core/datasets/Makefile old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/__init__.py b/megatron/core/datasets/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/bert_dataset.py b/megatron/core/datasets/bert_dataset.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/blended_dataset.py b/megatron/core/datasets/blended_dataset.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/blended_megatron_dataset_builder.py b/megatron/core/datasets/blended_megatron_dataset_builder.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/blended_megatron_dataset_config.py b/megatron/core/datasets/blended_megatron_dataset_config.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/gpt_dataset.py b/megatron/core/datasets/gpt_dataset.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/helpers.cpp b/megatron/core/datasets/helpers.cpp old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/helpers.py b/megatron/core/datasets/helpers.py old mode 100755 new mode 100644 diff --git a/megatron/core/datasets/helpers_cpp.cpython-310-x86_64-linux-gnu.so_bak b/megatron/core/datasets/helpers_cpp.cpython-310-x86_64-linux-gnu.so_bak new file mode 100755 index 0000000000000000000000000000000000000000..c398ffc62ef7822e975f481e70ff69ece755f775 GIT binary patch literal 247072 zcmeEv33wD$)^-OH2#R!2(6|SU3MiV+0?CIA2}z(@I}ivcF3^OeLo#L~>4u;pVUj2{ zZI8hfXLJM|MrIU69TZVCVRvT{WptEfM5>Lf$|8#S-*awN_3fhj8}<91=YRhHv1dZn z``+!`bI(0@saxfoeAmb>F)>|j`md|)H?|l9;F_)9?|j{i0xr|m6aV(JC2`zXDZcf# zyhJd@)s`SIl@Uj`wE|nO%e!{r0OKmB<$1 z5a~Ad5?3P~l|t!KdW!fd7ldDJx){baUQbRk(%()!+5MCt zLBHN&IgpeW9Li({fQk>$Q{{*Jd-WhMZ%O zj`aCcf4aHY#_8ontKTv`Q{!sbFqKPrkT|18Pq0DSHsTER(tRHtQ~0wdFI=(YuAJ>< z`GZbgKl+2~+NZyli(p%C{C$N#iqE?1eMy&=N0WLaEhHv3rtzkjfwyF@$g)_apwg>Vn7On%?l-X|Ei-_Pa&@&YYKC zGk0d6&F`N-=KdcW<~{d!)zuJxM?d9@Yqno9xb=m5|7tt;iovY|UOSk774GX=T<0(N z(`&BBdhI;tt5p&MXFcQ1(Esl(D5UcI}cf(3^l&0d{3oOEXQP8fDtB<-$mO0ErCvwTVI#@;@VSxmKNtB#Dt9?LdL({( z6g|&~qKBDq0FmvIwsJ(8S# zkxwLk3+#l(3hU1k#jZYx;%9QB_^ZpK@I%p#k?iv__|-^!BlH}J?;R!n+9>7zA&Ncx zGs<|;C5k<~6vf_NiDI{7qR8_l+C7r~UyIW2r^9|C>EY5S_EV2~;s2-nzKmjrt|;SL zT@*idPZT=|qEeCU?ecz+>+6bA-z`z}wmb@-i*h6B=K%;4iLZrbBFPyS#m_j-iX8vx zDDwO;>ad?fj&MDcI+u#-r7cq2+Xj)5K`*=Ktc`*}A?KY9py zj#O@Ylz#M16nToG*vXku#;;9L`g`{%c2XV1zkL>^T^>a~k@T|`@gvoDQxrXHk5aFO zD0a9eiu_BX*z-kE^!!Z}{?AeL@L-hrQfZX=($XmY=NqhlBI)N|%%dXl??>UEkJ2Bz zN3s76QTpTiNs-&}yD0hRM3KKw6un&?r5*dB-6PpgAGAv(-Ua1E;)7A_YH^f$T^2<@ ze}SBl^j{xE50j(ttD@N9>rv)`?NRbs8pRKM4*f^cLwyuKd=z?!RIiay%Kdi~JzN__ zZ(5Xmf>HYQE$By){B}Z=`FUcLa?g&^@A`v}RNwtk{MdO>?DMQB^NNe2)OTx?e(H}R zXKs}7C?-liJ+OX^q_>zT{qeyl_05RFe;B1+`=i)Z&nW%t?kM?O3*jTl|7MhN^NT3< z|49_TJqP1mr2Lbj%r{<(GJif8#cow}x=41_6vck_L}{1mDD#I#_>D;UpG3Kl`g>cH zesLm7xwl2B?+_FkDgWtF^6!i?uQ&+#BkBMADC>y(qu9xHQS84mik|O{VxN~psTawZ zgg@)=wkYHK%qaFy6D5Ba3W(T_QT+etD0*HTrJsHoCH{R;^fM$%yJtqJSMzC+2Ysek<#MW44s85izDzrMuQ%a-(^u-t1&Vfft!J2?}4!ixgGg7%}|OUS36?Ht=i z{rQPe_gl~(DW6S$5*ZBB^SKu52;yx!M0%P})9)biBmUqQ0$(dGww{Pe{LVvy_(tcS z$9kFggvUk31JJMN_ko^I(r$rE#l_~qeh$T7rk6WY=LcfEBYw)Kg7{wN{koji_Z$WgtEl&S z>c$u3R@PK|#urbm^oX#?A?}ifVs}|Zb#Z0I?TC2u=EU;lrAqde+K1Qj6B_iv#4CRtET2Q zUoCQQR-@|GC7uG0x4foQ2&Ko-1+1)?TH>xpd6gcwx4h0%TU8ScQ0OB~@N^yO;d+IE&r@rv|8C6vx`4BhM*5GhBQl}Qz zSD*~8m9vCWl$00OxxID872bMiu&5aS&YL-c_3nzn`Ek)>8`AqhAg4GFH{hpe8Evrly#QJ=N2^17=gR~woD9^$6Ki+vt z&p?4x1Y3}PUsmV&2}O;&{udZXzTSaHdb}m&`u*sdx+-=W=<1>?3A+*Z)Yr$?6_WZRWbGG@YepRJrU+p;I8x3 zRzkG=swy^j;q-oPr%8v?6#eBFI)zL~;;AdHCpm1?c&HJ&cDm8EYkXdJO_{q6we`5m zs=Sc(c4()-Q{^5hV*i4c98%|jna)6&5e+UkHUAeH2+59a8Px?wmvy{*e2RO#!<~r~oHb_ZZ#^Ym8nbe| z-nt@VtSzW1^Nd{#VN7GP%W&9asAP+=V=7U<~2xoc}GD@tZ@kYh+lN@J9EO-;#nO?B{Jety0&BdGUy|9`8Q7*76| zdINeXH~i0aoR0L63w9U#8sPqT%!h#to&oc%sdG++%}m9#sTSScO(RJDRJ|$E+#^IB z(Dfdti04Q{j-_Ihpg9(Gc`3KT5;DOQI2g1(d4C>G+!fU{U^}ma4O3YDjM12qO{@0t z)DD%Z<^j%K0Z&p)LKPxfQ7U5Q)s7jj*WHm?T2obAQ7tB5VIz51h&+&o1?bfZ}D zPb~$PdN7@@f(wh1O6Wh7T}8DK9~F@DF^k~@si6d>T<)@(Ix$U%Qk0Mm=AN2Xg~gbs zx*Cg}nY;#cV{uSX?QvGQs?t*PT~&iq)A4Uw))4v^{;n!DWAKpl%(TId3;+i)>8V-i zS%Zh9W)Va2QU|9E&PvT3oDLsZ#qrWpQd5Vdr3}fSEFGErk0UibEiE-84PDHE7M4*2y#Be3+#sjDm2SH1cI(^+5Sn@Tkai;+`r4!NOZQa<&_JuF7OTMxxL%bi>4 zDW=(sK8Pb?m{|GMBwwi|y^&!^;E{F382Ltdyt#;u98BzjDU9}2P4!T|COd-W=fB90 zQ`fEX(Y#BVNdJ80OHSTc1ZB8GOD1=%x6YAfuAm4~!8G&plo?$fvs%&R&3Lc^$A8Es zUsU@)MblRdXdKMc;YxVjH5I!o)o?Df91;w?qk(Vc_1fTSEMYvQp_-sL3J7gIAZ6*S z>f$P_{>e@0;f{2y^6Sbfu`|GX0%$C(u-s#GQeTmaG8|%YAji9*xYB1V+eM^Qc_CpG zl8P-3bK}5}j2xy|YKbzj)TpVWog3`n2;m)rE2?K;)rWX>)36s%%~K@!w4dAMhLqG) z)fU%zLQCJDo5SGHjs;X?E&=kZ>wQ=^q5_bUy1@uvMWq)WAC%F@N7mI;nf)lQ0yAdB zb9y~hH&%GdMRf~jO>H5s?JzTQ;3h{TlGFba=GbJCD#qdhZ$41Xc2K=EmtY#esGW`%`@ zEeS@zn)QUb>||s`vtY7Sh6W^ArqNtDyb2SlOK7#_DHU~Lb?1~!_f^z+L_!p9sta+( zmsiwFDj+!}g@jebS?|>M_AIds7b&OALrZu)xwH)fYt-AlsCb4Uh3KBTgFv`mYe*EP z5W_Zv|2M5evVvT5gF+%l!`V`y@ZUGqVX8Qx8asdWV$+p(v9f9~xp0?N7Ei;lUx=Lt z7(IM~r^H9rk(=(s)GDh42IHoP&YMSP*3O!WodxVIW=^Z9bk`S`dEg%OaGta9&WMxN zK!e2w*^uhmS?=Q6iv0Z4!4aZjk^@WjI>(K~jBW--)p12b+?c}VPpF)c@4gPBEanhJ zd2X85VkXQiMaspZjX}D5?8*0V_pS4bO-@R!iH(E`|IGMYu&5W~cc925=O)gElH zva3Vd|1?P&%+n(W&ucOUV`9UTBu83GdYU68D=Q@Z zbjlXft#n7);MCN#%=E!SveHtJ<0+G>x*A`#*vXdX$=JY@CeW4j#)hnQLY=v4Ik{4S*tw(0fa~j(TtSMnHxeSlZ;0nrDa|g#4;;*VBd>CfT$X&D-{oY)q^)khL6# zc7gR3sk8x7P+TjPu>UjFVu>bHn?qWn-SH?I9V}@S_Y^ns&PsHBQb=TV{se8(21|ZL zL;e`}AnJ;kGnm?loG1j$SKR(jOfpMN%nh)n1b=-Zpv_!vdwlvACIK~$;uYzpr;;A`xl3y4@f%HyZUDq6MoO)wX2I^X*lNt>hh3>4s61y?EmUtzD`>17 zNqZ~$urP!lp;UWAGi7>!c%!HIHq7wd(Em7383?_-cnUy)VDZGqN600*ta8 zUkvJAG-oAme(-9PhIgzdD{G3qM&EJOR8PZFmo5l$(qmwKYrb%nSbo@pV3PmH{}jxU$~BhFCfEc{M(I4sI6D;*x&hg(_<5 z*%POekz;H$e03NvY(zZP!(*xnvfc4TJgF)0fe^mjC2t|7;!!o8rwXj{ zaQ$KaiY>JgRNTi;wexK_CV|6=*t1@9ksGdV&=xN+A7Z*YtIu6VyV2B^FFZw%*2|B63KOYz0k;Hca< z{88E-w$pSONEYIHaO~6h-=4NJ_!=w8Az}%sC}%3G7ydm>zwctRGuMr;31BJL(~z&I zO%IeBr^_Uyh~xOZ@h|<4VjFcm6Za&gC`GV@&SIWoivRV&Rpf4@6s3yVkYsUOCZ*_$ zKcV-t!IAXD(p99TyK^{1NN3bE)D{$j{wK;0r80OC`t$y^agHVZ#2LXPSV!UIvv3mZ z99t_m`cZy&Na1HyRN#$=vuz8c@SCJ?-+2`{*%ohm(hToro34k~B0ulB6yDeNoX)*C z|6!c)>up=9<35~!f^D6S|H0wCY#-?OLB?m;KGCr#ue(&<8Mf>6Fp|Ia_#-&5)yK9+ z=NdLYkNTZu`&P&09F7*hMChQN!|k>N9oI4LX}eIzvpD|gwt+fU=g>*N(`;EfF6Qta zwoy9var$ny8+1IK!{clvI{xOIlQ^jtYpc`o*5_}-N#L%wMjd+x+>eugU2J|}`WfGa z0Li~i4~gqU{k)^WAAC^oQ!IR|9>3PYFVoNEHCXuCH$^^;7QV4f@bfHuN_q?jEU@tN z&KKwY7FzhERU)4y7QRsDms$7}onK+$r|5i}h0oUI+-c!$fA7KtAGGk#-ojRXn;gj@pf+-e$iq228@JY`Kd1@_uw$3+L_(C1ev+z^&_^lS+h`-RnH|p`1 zSa>7;3JbqXkH5*n@6`F77Jj>~--8xDd#R|eO}7g}&zc@T!NM=p<0o18<9hrQ3%^s3 zpKamebvX(xeA075o+%bSQIB71;T!e%jTXMY9>3MXx9RbhSa?Iv+bw*#p3hDTze%@q z&B7nl`QsM8R_EjO@4^^*Zq)NnuO7a-`u<}OxTY011th}KoD{tu4${YUJ%4bV*#!;aDQzZUtgT!@;#P5~( zgBJ^Xz~F9^`8Fy3XHxv_Qhb?TpxXn9BJ&%ie3}qu*n<{@kJtMJ#xGmqaWvliDH3md zVu}I_B_6LDn7@eiJzzQ zhJ6@83#9mi4HDOd5i3#5YKC%KS1ZewGw}g~Sh&_%@vOAQ|=98rSe~{w$m-u-SpCa+9#Aiyp`3?udTO~PVK3j@^RLVz@`1ulFDDnCf zoI@u{{KHcGDH8uji7%J<$0fd2;{PJ?4H9p>+eE>Q690k}f1bp@B=M~huP+HWbfTmm znO`8qUnAvHF2$Gmg;M-(RSLrTCj9UVl}MLw8F2=Tdx4;{Pr2$0a@}@tKl5<0O8e#OqTu4oyrF;{y43Nu9PN zi5D?MX8k4ppp<`##K%f}rp!wPWJ^32vgS{bcw>%8frS!}jREsFQR1;cHh)tj9t(N% zS1$2AO&05aiO0r;`D>8)A59khLgL8>8NYcFzt13XZIyWPO~!A5#LMqfE|hq@LTdh& zNPHiYg`G=0UNtj+DOMIrpUm@|?65n6q6^S?A8KvMti60=vpD6JIC4P#;Un%kB5`UG%*Gl{#iEog2 z<6TJ#Zj|^`DgHc(Pm}moi8tPfq~HY-pCQFxDDnR#@k=EBn8YuW_@NTNLgKHL_%@0E zjl^%1_~8=2N#e65e!IlyNc>KTA0hFY#OF%B_pk@!s#UncR}C4QR3@09p*iPt2)LgEif{BI@xxWwNk@wQ7t z_FpCO@e*Gx@d*-NBk_q6FHeAyB>s>TzrVy!m-rNkuao#piLaOVY>D?uydv>Fi7%A+ z84^EH;%7?y6p3$;_;QJ#CGoWqf4jstNc;u9tQeu+<#_y;7uzr;T%@hK9&K;knc{!xj~ zmiRwOydv?BNqnKi|5@TEO8gTNKSkmfN_@G*FOv9LiGNb!8zlZIiEotn#S%YH;{Ph~ ztrGvV#4nKeB@(|-;{PV`OCKTUoP>Q#J?=@2POU$i9as!D_)Lj!m-uXn?-COVP$YhJh_)3<{91{hDDmqgeu~7em-upt-yrd|62DR6 z8zlZMiEotnw;`?f^Rj>AV!zl%@p)hRaBx>=XJ;#8 zT8AGFZZ>fj#_cAiEr-Lw7fc+>c#(ThCtFpc_~_)Ny_CO(Vt3nuQv zc#(dkiv5C)RJl4eLG0rjZ`HUSVzJPH*6Vrp^!@=GrzKC&* zi7#e+=%i8qB*wc;+>h~Q6JNr(-NctNe!;}aj2D^sGRE^wd^zK}Ccc94bQAYyTx{Y2 zjK`XIAmbbpU&+{EVtRm!_BZh$#=T8^HRBi)U&Hv&4@Uh{81FK%gYjk)r!sCgaT?}GuEJEQ(n81FK1G2_i9p31o0#3hVhFmWm4MJDzzo^RqZ#&b@>VLaW$^B5PKSYO)?;zt-yH}L|-#U_4~@mLf8k#UZR|HRl~;>Q^GGx6h$dz<*rjAKmv z1mic5QfE)&1Nc(aLLWZZ7zml(fb z;(st+Wa8zF=bQLt#&b>l3ghV}UctE7#IG_QYvPrRb4>giV~2@fXWY-ks~Gn-@f(a| zO#CL}L*E$nZ)3d6#O;hX8yKsFxF>0~fN`GI2f$083df5r_;Cw<$bthF++xAAE!bzl zRTk{A;7Jxd!Ga4cINySYS#X9053=CPEcikT?rXtkSa6&LpS0-vhz0Mn;5`=nnFVjN z;P)(eodvJ5;N=#))PfgV@Z%QzkOc=UxW$5JTd>c9t1Q@K!ILa_f&~{?aJ~f(v)~L1 z9%R9nS@4Ax+}DE7u;4fgKDj8oe;u*leHOgOf5&v*2|W zyvl-?Tkui~UTndSTkt~`9I)UP3!ZJkJ`1k0V2=e)vfv38TwuZZ7Cg*?Gc0(J1z%>t z7g}&%3qHew<1F~(pDpcg!TT(Dj|G2b!P_kOJquoE!K*BIxdkt^;KdgFxCK9C!2t_y zvEbPj?6cr13-(y>BnzHk!37qaZ^6SXIKzSmS@2~Re4z#Rwcs->IL?AkK5l7$3*KkJ zdo1`f3*KhI?^*CV3tnZx%Pn}R1ur&m&UmN#?F2#M&#Toa%g9!?`r%fs-cBkTu6dvuq#Z9I;Xmh-~~_> z$@nekkaGTtrK7BzqnuS&i*9y;4b#nmAB1ksL1rP{#GWLUbyIzq0%YA>#pSbZUf>$b zy6NP0JXJS)1%HZeJ`nsVx_O0pSvQNB>HF6QST|STLdv)gJI16NZXtC|sF7xfjHd{t zhZFqG(=F>bOcxy+&pI7JY(FfG;Y@L4Ox^d z?|SMI5sd0N!Ji`U!-79W-n*EW<*fums`>&}PV&Bver?F|ljIeBHXnw7&UEC2*=N5Y z#^1Nk{oEoC%@-|ll6s|j62?R_i?lruQ#Ve8t<`RVNTTQZ(CLnGx6LOp167Uagec}l z6hlTn2f4Wd%cpQN_N`St9-@yfUBZzSDKE;*qhTZ zG~lj_z2$bgLq@yz3*r2>&cXymU5_X3W6(H?KT{ip>T55;M)DnBDE`Xi#Ng2ZNaPQf z$N`D8XZIl&1nA;?#$~cNsk%4=b#X2c;sh(nKuF>}nq8-EA}W;00;V~WeZ54T_V9JD z_8G3IQ}BxL6lI|lujwgX(o;O6r#QkxuPJDHD21x0n5(ClrKea!DWIrVxr%*L?<3_j z;AFx;?QT5rKr_C6h`Rd$ZNpGB_V3uYKv_^)aD9KV)rh9==qapXoi>GH1nx{u)XwH2 z4rrsHV>JgeAeK?^=QNL&3rOf?05$05?_+u|ajUiKW zGX0lf9c`|_>&dh&u3ZW3(d0v2r}(?FFPnv4;R<+@6(!J|Ok2n7l#VHCUUIy4D+g94 z_b1bzNYk%C--2W8?kGZn_8Rq(KxuM0Mabky35~WCn5aDmn|1}BO`b=x1p3Avp@d(; zu_%FClN+=*zoOuOByZ$G;TOqSx&nVmUOe845E2@t#g7L?nQCUC&|(vLG%V+!EpfediCPN@rYhkMJ$z|AcSZV2DU^_0nb` zdPjFD$PloXHrX0vNZL!w4G+>q?xkI6<#pM6X}zr>dL?>k$6ym-5`=2gOZy}&5WUOE zdhgWwp+Y(1F*iSg148{Jn?uVh{!#I4Kr~f9O+hD%*QRTvZ5S1CGnJwFHeXK>G7~v= zc$7dTrtV}^)Q58^Hm-g=>{ObdY?~K39?-5wT*C1--|7yB(Aru#!ay^EQE(y)yjQCw zmkj|DOO-1{|3G8CM&6`pZGS8x*68P#CMbbHE}AExII@4mKWyLc3GSbSW<3)FXV8Z% z`T2zs7&ezi(m>qpbjdpI?F+|!7e%Klx8}jBW;*aI{8Q0Ne0L=HG(!PJV z(tfDEl}T9`~~q!*8ASeFpEVcJH3+?|Co5k7N`1^qHaV=XVzBVb&7ft zk$j&bwyhv*vv-t&e9l(i&r-=ddwhky9iZ7vS(`|_{bk^(^ zv`!A!gR)5r%ch;Oq4}8-D8kBbllKDW20AbTtsw7?Cx!l8>ZXp*p$A31KB43Dva9VE z(?e#Pjh2>hpWi5f+yrsXZLs1Wu4JwE!X66}6n_ya`++x8QGJvjOmhN892BHZNFaf) zQPiQ7r?Me8p7LAY9o9+$uMy>PIysHn6*G{IO-A`j)@R{Gf>z8174=KU z7VRZ?H-(j}_+yh5H8;Vf?nal@zS@gnLS)_X6uRF){HMk>A4shYY8E4@ZvOoh6^;hY zYHyxNvoZ*f;PTfd7LfP~m;(P@00_PW!08{AX!tmL^D)#;csGA}qSCU@-b_!~UH&R0 z*yp{u!0*A$7v7ozzbhGcZ`qp{kv{x4A@ZjfeEoMMx>N*dzon@ZYURHrL7NOG=%NIO z{#XUd)Q&7f~68J$Lyoy9>cy+F47#(`sksX3)DTq8hy3~ z1FK2UzJo3xFxhxza(r+I0z*ygWE~je`TG*O zsyl+8W8UNRPe#Bo?U7Kx=fM>;A;RPmeaoGo-Khtirl_|jCj?argf|u)K4)@{dyU23 zc;>@aP9sl|x|IeWmw#pgS_Arvbq$@FWN#is%@=5hajF{%)WbQi0ot7O7@K~Lih3en zJ>IkzUiU<|f(qp4jh11#}<6EoDLUtY>RT~1Y6^s z$+0%Yp0}DNCpk@>!@U!8nl{Ab!}<~}wO5X9Qu-M=dY>b4TBeiEo4$&7sbA%+7>*(! zj!p(aYHt!3yp7tiHD?1_3()3lS`UZoh1$c_Ym7@P+YleWIj$>*To;Hv2v2Hn#(oQp zV=Jj1iamdm60^zvyKjgzqa&VTAGKO(YJ)blZmz(!>!>)ghfRB6k^b~#8XMLrs^%Ji zv0=X#oo2OI0(0-t-ePYzJ~=)|{lppY_C?>`VsEZSDK7P6r>bFedX)s|I0ybFa9vD+ z`c7a(7w6W0V=O`gc=OcF1%7m%jihIPFK53sN{@EZ+ojzKT0Vu??TXs1>BMl~8fQ~m zyaGK4dvI*w_Mtp{ufvZbioN-3s;U3%9O`tg9-lamZFOCG5x1Aqp0`y|&vyF9Lx%&t zg0XCN4fu#m#goGsb@YLJO>1JBjt}>aY-(d0@H*9Ym%6$DHTK(KIO;#0TX)mA7?-fm|pV(<|(oXjTnx-l=fTVL~>GZL-Cjpg!D!Jys*Ik zY;q$t2?UQrF`b*7{#%n1o!DAPQnEUHXLXD)?W1W8sL#9(rRfe#!4GP;p;MDjq%(6E zw0IKq3Dw#6p*DsZRwVF!KqZw^3D8ADqSlq_2DcB2_DPz_q5v_Hbb8Oze1KHnDqLy2 z>|3CI9V~>g5jjziz7C7>29WY}i@*#RYWrfR*Z?S_wQ~T%p|Ss%j=9h_WI?FmHijL>j?8B>S-$0`^sj)E84P@Mdf=SeUv@@;l zb{wTC>^Q6)v|FT-=cDAX;VhfaKTUvpfc!KSag`i0*Y!VOO0^X$dRp1$se$7#1<4Ha zH#*w#o8Pjx{G}@zBEVa8ytaPmAR=ntVuDAL;--eg7<-Ec1-twlqWBB@AI;8*wKreH zWI4v+t}cHU5%EdN%s&xacK_=FeU6}Lblfjp{!Aq>!meQOyUa!_B8-tuyD&z4i{;MT zuj%W*O4Gf`$FYfnHPE-dj=8%!X+vkgyYOxHmLZg@x(kl$F03Oya9|n#kj9qX!onW= z7m3IBkdi`B5{&|Xcn=vKwkHf)d?sEpu_s*WK)l5XWEcL&ZMc-#`+p|ZVg4B0zf1Go z!}gorIjqf>zVDr!fYZ;;~8-)Q;=drLB{;sT!Sx6nd|<~_99!TC7H7RS+! z-@E*Sp1PxpZDnuLZy@9H&&kXKq&sMmGNdVCbGw#}BoLLvajI?HZL>9}|8?6z%7)w_ z9fWM0lWDWJ#G)psv}A>fDhfUU8TDM>z>YVj@NAfR#&8Yj8!5`rxGl5rNVT~Qg_%vk zHNo3aLfJ^jT6^PQ@Q;Y)@(ti^C)Z`lQJfu|^>JEyvL z11k|a!>YA~x;)*w)aCKG#f5wU0uuFr{FX25f2gMks-FeSQ@_l2?4xaSb-8G&mR6EW zZ6OmyLNWl#(HIXD|8frTb4Z@LbQ2<6zajPjMN+zUL&_y014@bYmMuw;{BNs~Q6_S?NvdVw4z`*p;=EDrhV_7n7=jSK}XHCk5)a zuB^qRG;b3PRHY~?FRMIh_Hs#-W*@@yvoH&=KfwY9X&Z_CkSptu{lPY8)_e9Ate{oM|fPacr zWBuaWtu(D-3v&6FlHTmiw7&*XsctY5Dh2HWvs=oH%F01~TMcvY=QrY(nhl$G$4|%3 zp0Kc1{$U5SkgrGeLf+H+b?A05^xK%PL32u+g5i)K$oMXQ!b}c#SJ{v3}(sszp zR;f*clRzTjy|NC%i(+};XQ8N`x^9Ft50RZfKPzxSz!zmI=6SI@6#Mg7%|MxLF{Bc# zFk^SnO>A$J+FoYAdhP99yuU@Jq!(BnsUU{BXLAjj9U1L|02lV6n)(CF5J-vH0-RI=W-H`7YUr7kAf zo$7mv`Z!lW-OA;vJDlpjv~3?#6wGVz;A4$8$HUb;hFr@;2Rf0Ac1tHY;WpE}d z6#7t)2YH^PZU|DAJoAA{gIP3^tVeu(>pXbjZWxBSA4+V_jrr<^oEx0#`!p9lDQ)p# zC8YSzl*Se;avW{`$C4YNME||wIgfutl2U?J!)}U_bCV+MmY`1eW2LQhl=m z8i0yd-<8w+slBBi8qEIkTlSYf#pVoL-&oAv*$Eyf@;7tVg~nCuyBUqvv^J?@sNSy4 zpZae7sfpv)ip1f|346T>jz<+zbd zQpNaYYCuu1P1f4U>Tv{%i{+^k)zd`-VQAL$SS`((XuE-ONH8kixryQ>7*(}n&a@|T zDYyKTQU-G=SN?=jx+9MEG4j-2+=_mnKV2;#`_K8w_K#k==ScY z(d{gcS_wwquqg)Xrru~L!{Xn7SwypVIzmGhpGS>~!Gfki&9t7=Z7omjnasJxYu(=! z*8cnl;`%hMoFYDt?7k1$$nYOz|J4?`R(xQU>vCN#sq9F7;C!_ORbw211+b;fY0?5{R3FjNdrI6UbpVdojn@5H|)F zG#A~va7#bjIzzW3L$|rcts0kyTg1yTm<@5i!7Y@4d0pHsI%^*gmyTOphT)h)19{0Q zTIQt)4P5y0`?TOa;KD9jilTn5O@EuL3(u-3hdjNU{&W(fzkD180l(#=W~9v&E%_3J@z4;+ zc;-FS2OE&k3N|9J>8suFmZX|s9A$%;u-Z5F%2f}N$Kkn!3KlOC6 zHC}&HEVid*ZQRKUIHrAw9a&PuySTC@25Yg-6PGh9ksXP$4PfoPG$? zmGKfHhbzOmQBub1=uxLCqZ{UMsA|n$NYpcq|5;>_`38)NmC;Tb*`Bk4Fob}K;LKi} zf}*I#vKr2~?Rz8+^liqaOs${HowqCLgOw53Mqsf~&Rz{zah?X&dm2ga3`lV3DVqC4t*!vcg(epB)jAo7ycLcr8^rHta3@PLFjaFq` zvQARQHVhS~Dr3jztc;B~yn}8Ma|i9fVLb^`Av54ka&QF=S)z}uFH>_-)wn{GmBW0P zGUgwJ2!XzT#D&x)E2HcyX-D}S4qDrQy%1?Ng8}M0*6A*nif5P^ z@2-!r@u|{G4B5liV@voIu^vaUjRcp7{Uhz04J0g{-`$$rUn{!+#iBeMn21dd`ms

}AZAZM}8#`Ev3!4Gv9%)x>~k3)Qw4(VYfV6_Q3)u3=# z5)SVUhkyHTVYwcv(DV4M!7^PQhMR&@!=>#Jp4IZOP@Xvi&kW;&pND7vsTs=E`geHf zf$-4n;h}#u3+J@&g~QDzPjF^9=JfvV<}vV%Q)rhhlQIfk3ojQcPOMk0@mr}fdi(tYlY*Pp=vIsIL{DE8CB08d?=;f48tJiD*FAVY zN!BXqj{Odo&*63dA$;6_2v7YF;b;Db@WWKKi1N{f7{8N7ir3zTFM%8Jo`JVZy1Aj& ztipJ25{#r6Ks>vLS+%0EsR3~YfX;;C9*YHmB^BuV592X*e1DT@fg+cGS^}DZj&@z7 z_%D0_xi${9d2h&p|BVk6cIK!!I)eig-afZbE8$(Z_>CcdjJ@7uj4`k!O5~tKJ@rIA z>D0wLW`x9!Hz5~q$sNqGwI0xM#=vWvvq&ZH>bR!ZPqQK#4)`JD}b42HVHV z?iF@S-`8vZe7&=io?e>vO;4rxa3Ic`-9;M-qdMR>arU`4MjqtmJwuxSWn=d>))gq> z4nu<$PD@Do=$IH;A5D{^>Vt&)0b`hqYZ57p@SHoOa?0I!l z{N?e`*KFIskhZQ|MdK)rp&TGxEq$%C6X#Wq)k7_@+sGl%tR-IQibwL9e9{nmH+YbL z4UYLc{Ww2+J9J1VQ0uURdd%K@AzTnNXm5TD(KI_c7!7Fm0TlSVyU>5hNRXS;kNExV z&7V^VI6~{9BeX96ZP>e{V=%Q4GpF%4B&IWFJq?=8Xhd#N9`o00u`DR>^&C6m#DhFM z2E|F%t8j7zzODd#9&-Vp)5BlYWl|=tBvOpbT0V~-QMvWm(TQIIwzf{ zn#hOh1Gi#sf=O74%Rhxql-ir8!v<)Mj9E*H_AW_ICZr#(vfp!CpVB4in z^1Shs$0rmobA2pa7WZ=0YPfPGD8x>*-`xY*t~{3#2QvPE%{=N&&%KJ=Y0p3j&^t6EDu7ZP6ZG zEmXqW3)}{%cQ*PUoU`^WH$62Bb}v$bK{S{?4%OfU^Z;zb1k<1xItaH4)5BpfuzdK0Zj{lk3q}+5?jyY zQQG%k%#U9ts7H={R%BQ9H=V>@+1+TDrjsT1yPv?#+>?8;2Dab*67JOw)PL?tJJ#0r zyYB~~ym|dicV~XIt}2~3BRZ0ym!=a)Np?E-gh)wypmR@%L>E&cbrVg5j_p-9Hnqnp z?O&a)%>D5R6gqQlUSLXBXCOZzCy?H244+v0d-rRx;v&VFOVH$e>t|>NctylUTp82!1(0OoTjh4 z;C^l=2{rSLxt%2F%rQ;tu%o7B&l_N@X80Un$n-~^o`g{+!x44;a6QvWh2*lIXKE|KKDW1V*VFUx}n-}!C0guI}0e?fx892 zg`p^cQ7Cmr3>QW-r5!K^Lk~e0Qmmkj%AD7sR$Fi)aL((fitW`{!Y?9+oYyJBt3;|D zL3-v#)?=IiYu`5){e1si_@+k$*{cg^~3ji+X2z5C5INBZA--|8b+ zFn$6%!10nz$kd5{n|A-;Y}dNDVpd}Wyq|-2c2Ndkqi(xPU6Z3~dG?oGH=w2Z+%R%p z&zQVGerMCZE+gjkiBX2Jfh<*qv4Oan*1$k^&O4W;Rk~*=YlVHNN1QOLgb{&sEC*3$ zpJZVj4oOVXGB23SW)Yq#`-ebnw?KJZ@Py$n0=3-(<=ui`>1N>! z_(Eo}5nfZ*NAO0&D&S3E1XBT5QS-mSHt`ISuOgFpgSg;hfb#4d;|SXaz2jk8MbnQk z2k!1oKf)aT!cF+2Fo$OZ5a#e80n-c=^)MSo$5mo{if9kLIK6HUy$O(ah-?pR42~_V zVmu7p3J=XwR3}b=2NZO>VVlNxu{l>5mY|P|rad^1?Q|k*wud0>;q3_a@J^2U7VKgC zjbsluj-1!0S6;x`*%Tyu=w-2oYju0Li0y%28-e};wQ&|f2SQMik$Th2Y=}7kVxs-s z)c)>yBjFZy&NDMXy9duiqnZApwtKLPgvG2}>EuZQIS5uV9ZBx4wY?%r;#PlynmJgD zq@w9pp{+%$zeKHW_R`5rk^A=m>TSc$|mQsu#wk)N=XzmFyWdif42 z|2$Fts+YO^cq;!@lq%k&f0Ify%WtQ&{=R=7$K^i&F{r>h!pr}fMLwXd1#OmZ%(2C2 z2w#{IHZkgt^Vr5=ce=qA4wQl$e(}M4YM`YE;l@Zqmb5%SdQ1rZYHoe=d*RS|TeFXYJ|Ov%Q(V?nnW6wSXXf+*4Zzqk^97El&Yq zRHCyXC?-+6b2;UXvhYmofOZj=WuUg_MOi#m!ioKEG1`07-ePJ*k6W&$*5Su2;IP7= z*D8O{5{8{NzX~Rwf|Z~+Ny_7-_7a5V<6NTOX-^T(hxuu*Ql^13?q81Pyb^u~=JW>c zVNO}NXkWd+!SxhO|ItJQKOut8r{H%G9P&}(`S}DowHjIqDR`qUtiiC?s03WeJ<&vT z!qcXG1YuCEo-o3eExrrz97yZR)xtY)YS3|%j&tHn|BjcTJeq#4&r2R?dzDUS(%$8Y z+F4U8s!JUX+VaD*X{Z+SwiW;NUwHU|JAQ7INRc~EZ^g{pm&+RE$Lu?_q164c@78p@ z(|etGua4fg%GxxuUTONS6X$;MJ_6P{cqJ_f&5W>)Hkx132(=zU;e-%Q3vbrW;-;bC zMLfRJ6xPL4*hk4+uxpE|U_DCqjY%ku>l^O|T#nXK^Z|>UYaqk_51MYTCM^wn0QcI< zuo|s5l2Uhsw4qsM0%iI%LZ~m@2M5C@Dhf^#1^fQhu{+$Ki1kIljivV2iD4~+C7GgL z%fEb@Rp=9GO?PQdtm?#C<1C5=UEPL!E%thJR7w|bH*^z}k4?LT;NEA+ z`7}ZS!vFER_S-@)o#Biv?dM3E6X%nlJpV|nyzr7dUS{K!^9A@KQEfliD;7eAkJ5&a zt-$X1$7?S5j^jyKb74?e*$-L62)4XPk0*|4{{>;hn%~EGAmLLIpJuu%x#bi53r8_atTRS~3dLz;hZ4bMTKf3f7gb2Oa4Wu=;)l`iIduwiPMDui)TU zH9Ehve<6H}%Rktq9(2WM-X8o~?F!b}Rd^-%qHs=wQxKzRr4_zwF{k=oP_K;ML6{59@mg4^WzoWIvSVucL*6VE{+|c)U@9AFZ zz0lqla7X6+1|J;gdnKCMP~wpyIOa@j_P!4z#Vd_GhYZstO?mu7wfAArB#qcq%f^0C zoA*54^1^mgpzp_c-0NuL)Y?;Q`@}wiM~jX&r@BW3i21`%Z3JW%)3$h0VW983^stzY zU}BNKR@=(PK%e%|j`KVP_CU)&@im;~qyc~5_Z6Poy#lc*;u9#-utwe&!T8>B7VWf2 zNpB~uBJyEw+5_6#NEC9=lw4O!gxB!sP|ogA;!w_Ek_x-(&ZcXVZQirB)v$D^V-V{H zH?b5*meD_C8K=WCGGQ6?W}fx})q!6lJAk&JM!J}`_Qe;P#$!^h_a}?|E79O2|CuDW zKL648?=Z+q_D^j_FQIf}KY3Ev9S)Hf;?!>=IWq_uhhpgklG!$G3#)W?JokSv30fO7 zlM?Y#c@hp0d#h+3f`)EdlcfEH!|u?uKjDT>7~`cKGp6s%jvQm35s{tp`+R>wyV8?n zKS^MDpKs0*QYk0!eE>=Mbw2EC;dQ=rA)Wp<-^<+TX7kH@7ivrNmRyJ{J+{E0zbAMW zyc@|2wayc~58I?+t#Ru6e0}~V8scF*t1;f^d;14;usA4Pw=7F@T&Wk{gyv`ei}z#n z#}B7AN7I}_8@{*j2s$O{;7>vag>;x4N8|Ax=R~~rs8AHI9_3*XMF~Y22CZRrKrb~u zPx_+0Z6JI`BCK-5$kc9`6!aqwuT8W=Yjuq*|l(;cC%4jb|YJocnVW8T}-{YQ5cQ=l=kd8}5s6wF&5K7AG=*It|nZr8ZcaT8yJ|+1h`l!oE z;ezIb&kxCl1Nr?$r*A_-rD--^3%x)a4|^aLE@1u4=GQ`n3MU~-JVkNnQRs;V?3uMB zqTGxqHk&t5>%~#LUg`iL4PR+dN-8OZx-9AO)5Xx=W2XMJcL={9!mkm2Li-T1idSpf z{z~(zz1j#oEWw*i7#P0PW}GlWpVUK>5h@L|WJl~|G

  • Cc;@&6-gm%iqZiG(o7N{d1F>+hco%{G3ji+v zjhK!(Tv?lDoLvwYi`B(P_U301P~a!9X~y+hFBHL>U9`(Z6D6AU;p;0EP8_zSd6E+o zrUaTN(G7-Gb8bXC^>Bpr%#G!u6&!tQq7rr&AxGbHJdweHLFCu*T&Pk!-vN!0q2m#> zPtHm^Bn;+p@5lUF#{~xF`Ws)eqQwSGZ=2Q)^>AU~2O)Urhw&2KCW0JZJQ%aoOsW)D zQm+{=hu6@2D?34#Z?oG|B$cQhmV9R5_HVbv59@h{WC&y!Q zOU3G|RGR-|Znj@P=6XOo`X;p(ohSMb-idmbmt3J(CiFcWUR>`>@BNt+dq7*f4;>;K za_Vt1?$o{7v&e}*3PqCaIBI%%vErrqLmtg$vHHlvG@id?ng!QITncg7g1b!T5{Sd( zmc9qun>dtRL88fd_`c}aOLj=Pziiy->ru8(V}48KS%Ad_>Q0TZRh#?-HaU55hdMir z^_}9cr^jyvDmE2k)@x35e#?Z_nYC`_0I`}SW!*?x;qU#V5mW$g5wjB7!NF0@VaK0@ zS+t=vT|AmlIsrQSN_ya1MejFmu*J>rTuUW^4J$djZvLz!g>acS(q&~ zi1mHbYz+CBFTwU`y!wVJ=bz0({%K+ZPbI2pHV^x=^=QwDXwtAh1F7_3e+73KF`y0C z{!XZAT{(S7VZIZ~1={<;vs-yKU~i%SBoB0#f~KQR74#w%?;DAf_Libfj23OJ+V?Pq zU=@Jy%UTEFMiYnU>&vG+DZ@2lu-l4;x3n}zA zgld0iS?$Y!!s8 zSD#rFSd5VddxJQ`f@22pbU5VNWN|uV zEtylOd;IxTK%ky?>=e@ z7={?b^n)oRUB{Qu8}^a;`*5TW9%u0pILzV)YM(xaA~(?EIGg^7H+<@ipD=95)t@iq zCQ;}uFlYUF$^YZ_nTa+r?ekMuOGNv;A2#sc+2!D`wZE1_}AL- z)h~4RD`-1uz)sQGH{!KjT3)xb@t^}!#xo;y_Kix+M(*cd;w4e;>U5kF(d;dB>grWG zl;Zd_-vNocVE1Q;D{GH;5Y2GtYt8T_Fp*|NK@v==o9HW2t1)}N4sVZZzOS7D*N}J@ z-ta!hIRMWdKHzkRwdrK6`217Mp)x022f2*5$Ys#vt2eq1xeo14Xb}#BHn=CzyfmLY zM*lDyZRB#p`~P@4jGUgVRUx@T4=?fbBoQAwgC|e|N_;Q_J|Ltvt1EaNuHbj@{aW=S zvUfG?H02Mb^jRC0XPCnbbtiZ) zr6Yt*YhY-(#}ySMwlNLMfU^;HvC{DOEcKe3RvnT(Gk8=f*d641(1!ny)l9QU9F?c7ZFg`d5m2|#- zTvT;V$9CB7>c^nZ=N|lT^huvWp}hu72E})q6!k&=1~R_y)^t047O904;lHjwhKQu2 zBXVNB`B;7@;Ek|u3g#p{cfm1k^&I=zF?cf$LzhU8b&<%>mwx4I;HCZ-xZ}&4tUnI~b&L{dG$&HW{EsI$m zMLrX@h}j5<@Cd8>b&j9}nv+`*QLLFhLnIn-k)!$L^S5*^iMSLIZ#i>edmc~f(WX$e zT12xqcm9Z(FeJ#)M9)NE<|ZO&BVvthZr{_6n`Xn3iM26d9rj)mgh3&!sMa) zQdLZOuz32BS_BH1L2ZQx%%h<|U4q~pur@=QdnwXbL?U$#2u13^HVCa7*L1upuh)$U z>Hh`$`x-S8_BR5p4%;~VsA+!(iShUC?O^-6KSkKz3AnUy`-?|B+5RT7-OBbi5{zVj zH=+zOsX55Xu)k{N*#5TvfoQA!-G4u+lkM*uZkkB#J&78|Vt-DZliNFkxv=)WM2}{+ zw{Cx@f&az!_a20{%JZtu$?_~_E=(S(uhstUA`Yf?w3V(qwD-{wquSqQL?Zk9TPV{1 z$o_l<ugxM_p=LL112DcN!^hHXTPp_rg6l;zK&Tq*8yJ`z8G19Hj@& z#T-&L2bHwn{~zk!JwEE<`r}R@k*MGXB^vKhg9h;us?kJ16G?Pe7mIi5m5Ng86*W;* zgkUzvc3q^3)@!8~tyWuY>jm)^u0gyZRYWO@T4mN%LO~38Ah45J^0xdLqeLqsUSBysDob zp8UD1Z(TU?R2j)sgR?+eC8ZaK-~J+eL~+ZINqOYo0^OG(`2)U-EkK$OFns&S~~Bi!s_zNecp-$ZJbHuge8h=&+N zz5UEdN#G0|!%LpmnMKBbkqQs&UU;zzM-ut&p2m{Kz7Y<<`J=w{pK)2Ah8-GRoKEGO z$7~CJM9u#ael8zu1O8ty5C04JvxWbA_?#;mF4}!X+vvsk3ZOtKV@AWDgGxg-KpT zGIM1QVZ7?a$1C3hZ5E0Bs^y=a)#))@t&Q!ghiX=-;TBx6@zwPgAGInGH z9Q|LA(mk;NXq9ctl>JOy-zLDX*_7@8Kj-6e0sftj%LVv$;so#?9@!?q<81P3V{Rpx zY;91a0Ms)x-1vgj=$!D?IpG_!;dIOwOy3>^rsb?}A^cWv75F7FvD_SO#|320BJr*g zzQJIKjjhMri5Wur12X}bv(zWhPiIup+}xP_Wsx*eiNm_B!HqPbS9zQV9k5WJ;nK)6ZRDIw!5_yL;Dx}}&i!C33?|M!(wkuGJ91(_-50tN> zl$Q4r{l*_~zxi`bomC*m+&AdMI-cQZL(TF~Y)_f`;?hl}CF5I98E+%#&&5P86w@sQ zXBREQ%rM81D5PXfC<~WADls0k__Twd-O&ZAvyWteToxdEKk7PW+!7r}Tc|NwV?wb> z)Dy&%MQlR}wgrV4l9+w-Co@H%)*%9U@_Y@q_Jv$xV~4H?iQj@^tL{(UOkOjx z#hwMVj^iVT^5n;arB%nUvOVQ_$H=+s18+60K8no`Q8J2FrbloiE75DnW0LR9_}V zg5NS7C(_au6zCz_gOWva1ZLHseH;afLzgoAg8E*~Ku0j97Usr<+r(YW5!aT;%kbjW z6Bn!whfTB9FmD@y^(xI4$ooFXJ5R{lI!qHJbdsF{c$e!wiXwY@+Ak@b)VZHCS zRYeguL`dh+Dd>Nu-)Mt;*IUs+gp*pc9Ftc|(VSgRq1Gp8CZ&JIOgDq6Fm)h+yLLm* zy7|)^GJdrSRO@TqJZT#rTKx;Sx|TQ8!qD>+zMgPEh-a#|6v?Ia&)J@LN3`GBxNRX? z+^5*((QVhc%BO6NXlIflM7x5ciESX-Q8t;wsFmh%Ef00cS@Q0sDjC;C*a+8!dDhT2 z2sdQ*jCrZyIpK42!avUmU!N1cHXBY)&ei`|{>xoffe#M0fNjwKFc*YSHA)Q&M)X>E zB39zMA`fojKU0gAP=)>(AwoY=`7k#a1^Pi|Tky|g(l@JvQpBNT)C$^Lv_Dxgnw7+& zyMiCHn-)EJs)IZ?Q8Ju7Lr=anqT8eDT2-w;SOL?-aB(9d6njxQ6UxMPLUy;>^qSl% z6_DpkhucEr{B`C_loM?6pse|gRa?5$XpQXSZns1ZCuZhh?LFXcrMaI_L>6M9*rkPP zYZxXAbfbk2$)5;z^47{Vfv=( znC@E~joTH-plPzg_Cg5eCZ@BK-es28(|&L0FjrANvrq#<%DiA&BmOp1dSA>H^w6y{00~Rz3gsCdzl1ka{;{yW0w*jp9Z|BNTIBjnU zw^GR75}raJxF!4<;rJK972!AddI55qmmZkqVC?;1xo!^b2j6eQ?grsQgu}CfOU3pg z;3x6rE%Ulv41Fn_u-h|3!(Z$g{CN!LIb7Pj>m@_f{6~9j=5;uz_ zuD|0VG62(iom0MG5N7kg5aakM4l(YgJcu#QaYUj%T5e&pX2Z4-Vazt#yN326iSzC3 zhm)5DvwyO9AZ9;Y^KR(Y51Foo1*JP^_9O8tiWb&$N)c%}j9o}Kz=hm<-a<8)uA-C< z#ij|&))jaSCZJcDeqd7&Pccq3vAI~C5ZPuES#R}cTW#V0fV}A5N8PN$s6sY@+^g^N z=NMGwR9189Xe!jXVVyPg7{o(kmN()3Y6!2uv|DKwzXW21mH|+;UdqS{TSnRyZ;z#t zZqw${%t7HCjxoF9tl6R+;M*+vEOh%>p7{jS%Fb7KpqF!7HXaz7a=c~x&MPmTeUO{D zz3G5>td-*h!5KgRjbj`>GOyW~U~9~--M`s*zwSMXykgh029SsfR`8!~#8!;yTViSg z#?<66rlLokrj(zX-3(OV*we)Oizw1OxnIsX4u{g*m->lE%Z4;MV4E`QWL=k($itMB8mcpbA0K6b-n);AXh6Sds@A~8*;gwd;xh7`vdB%(aQXY_)N)XFO$wP$vCYkb7qZ(i`l*9rj;t;OF3M z(_1O>=LoFszJgwGDh;gUN96D+&@gK6S!=Gt>@JSQ$u-W2NSV9R)p(hcApLc&%HTQ|K7p{?fw^-x(j1NT zSGu%zE6i?9Ze+{cb*%~%0b3~YONd%HtUBKsMmRf%;9gexSLQiVgCXqa0zA~DTW~cU zP5Z!AZc$sikb$1;3+9>Ge56-&(|>aQ7jS>uxU1YesU|PfidRwO>g3P^N}PS}5sZo+ z>*K&z3en_9DERsInZtceC%7h_NRJRGkCdE+-fH)JTgyuHU1P?C%~ud0u@IOIP##Gc zsp_l7eathyhKH!ZrW0FeJ=R7N`v{B&d<9R~3cyKMfo!1mFxUIi58BesG@t7xnZ}Zf=Z&7_|!cu1C&`RO_wtY-Ubk>OqK^;)npjK`{HP?^xLlX?6oD- z3LR#w&^bAbsEs>Zcbty=7G6V4qVTz05cHE(7f)ntaFw$P@Vd9v*x#h7OF*J2#X>p# zD5Ag_bc$^w)+4|)&`%vt6F+fGSH?Ify}>!*DVB%Wbxm+JM;c3>oD z=r;2j^MfSVggPS+W8!}kg*)5A)#iAhvxXaHqt+w+TAOVv^$e(FI_N(K(ldG7iRNnb z)2+GVwrFnJHkv!a7Opmb2S-zDfYsdVYCXx-`r1~trZl`(SaO}H-bSqreFajrxmmT| zL#-O4)G$WE9PE3uzweEG0@IYcM&GsNs;=9r(Z($rorg%a<;blnc8wOP(F4?#H^
      (BKB;l2-lDW#jaQm8*RhxJ;#9X{Z)qg`(lo^lGRQ~|ZC91LgDmx70 zZBtbPwt{oFEh-z6TbW18YeQ~uA7}bf3%{q z)WT;}Te9ld8yyAs5PF%~^j&rdTXw-Rh~^f%c(5;mz*?wU1MnsUfolnDi@u$$S7e-z z_p6HS`;IcWb=L}#7gnIsp33qK$B$&;f8LD-atj4xviOCiXf=>7$kcvfnSK|ZSb;N! z6M0Oym#GkBhz6GHpI}+@5k=8tQ|ULi$SSkh*j41MzHk}!$4^Fe*Jvz-V$<40Xy1Te@xSQ;Kmr$!*&BL*) zn>p+{`iLl(A5)h=2fV&V-|5Gs5gs9fHNlkPL`aTBD)p*tvZ}`oqP|_`BG#(%aUfE% z(8g_6?iVgo4?4}Avt4qgn%{P>=7sUDnmu!>`EjI%WBT;UQMH~<{b^3ho;TO zS!fRR)f7^VnJMysV#9)Du@-~X9c_K;hRh(7CBrNXon}44oa1f-y_MH$+^2Prtm1|l zWmkraA_Ovox-!H^fK{1`P8Bwe+qgBlalQv%;wpZmzQIcz%&!*9{`R}moFh6!mAN&1 zZkJm3QC#ZdMmWVJqSD>zdj+VydSC;)vMy|3{~%Bh1P%)V{eys9?fLVvvzX7pjBCb? zz~nCdej>Lmjq80FypkZ6Y$o5?uDXT&FptqfI1CZo=ql#vTb`Yj zY#ZmsAj4ONS}#-d4b~$Y`bJ+Mjteb9(vMW$rU~ zosJ_8b&lpVKXoYRA^k5j3R4In?%RxzJ>g>igj{pCL+6CDXZ^+$YAG`JXwQa}r;fH| zo2e(yynyyTb%k)v^6%zxeNV}L->>h7GT-)one+Q7Bt|m&b}S>(KrS;K`0SNO(DlYk zEl+K}hb3YY6iN_~8gk*ktfEH5v z2)i=fke%u&Z0?W^RUGX!%oP}`P12LP+&sIJ$IM{QQjIWXX=@0UUf%tas|h-nQe=+R z|Ku+hF^Jt3qAWdN^Vf?LfyFyEny3KTbl=@sf#e|QoL&||RYi_(MZM{svDYyPK`fH1 z^N3nacQx}`V0wy(aUM}d;XLy&i%*1^wa$Z%5YR10=fS%y6w`Xo)a;y?CeH)cQT~a+Wy*VL5$r!UU}Jf&YDbD7-7wNOfdrA3 zL&|vpc0nliFqPY-_5^{5UVyPGQqh?ardT*GDzR7~#0%SYvM0&rQ5eZ+qW@p`M3Q%R z=7%SEzgyrwBIxL8v6)OZO@F0!xLPf#YV+X5ILhMncr$}ZO~pamjGg|_w+nlqY5|Y% z8Xswqkt+)~^y!iJ%5vzBNo!rl=RQA}v_Wm_L&b%=L&CU7y*~VSIC*^K`cPk9 z#)|hK40$-7_iJvMNmhd62rjq5TOQR<@|I!|wD>Kn+*jCrmFg#X3&%MiXW!#T^3N^P z$)R6imVU*q>9^Yd!fUt}AKQ;FBFJ%bvd71?pLapN-CtA$pFmWTov#$|oVW~B=EyP=toE@;2LFaJd(4%{(n?PsZ z&F;FZQgv`Fr{O!7VG@(I9$pQ_m|fp3Bhc@fk6jFn9X(Uj?=p6tkHYOOkE&~|I9aXC zp~+EXmR>@Q$OTNCe*|M;8O4;zUF7WBQcSePXWE_GK`A=0GaZ23FiGa^om9^m(4b^h zAKR~w%vWRFK=tIbP*mv+wDp2^0l#*Wf|`7ZAx}7d&r{ucK7TYB{!N!l)HIdkZPn#9 zaGF*-*r)L~gzSq`@sE5*AGh7|{Y@4&*=C1vj$HWM3mVUsX2ylDO`6p1!?c9xb$?Q( zVU;rR3H5E!5;GsV?oGYj@=lfwe@#ksKMK8U6Cat?Ouf+M5=Na-2qzESjug|ziH!O& zOD8khv4*s9G2?~ic{hzp)a4_$~ z%Bt1B%zXb=J)SGlG5E{LrWdBB3t`?!<3y1r?VrATe4byr&n@4pZUVm?+RPh=02 zw<$VNk=nc|hyIdYr;a46j=7wwQAvIAL_o<$T_~{a)uR5mH-wVE{17n+Q{M^-^wTWk zKu(1@cxn`T#wpYeUu6zpY6W>~KMiI{@KvBOfIf@>RL<_mZ0=QGlS411eRIX*gcF5- z?xKNY)ovc7voj#QaRKQLq&v2^APsaNUF<=+jfTu0L@Lzw&T1_=^r0)LdS_Lg?DGQQ z*=TrFePwRhmSThTs4KNW-4K~S8hIEIqe9~E3(}=@7Joe>aI<$s+XurEECi!@K7-? zeXS}#0C+_aUbtBWw4eV)q0ClpcXso}d44~iZxQMv^Ewr}wr|O{Edvo|ch!To?cIIb zbvbSCt|p&i@Y8pCa1`+n5rW}pTGk>uoZl4uRtDrFTvNMcoBD^P_$5uQsXuRY-CQO* zO@*iU3O}DfOMePF{#g+4)h(h^e$)d{}a`AxmGISbzh zLCph$z~MolZxA?IfIZ`D8t!XqP))7W6i#gGG;up26MPgbNUXdWOh?2E#oi~sNz+9O zgBzOf&&~8@j<~pF$vF=YldQV4(+$lF5b^*Nr+wS>gNk&R{eVd`UuK?@LN`CBU;Tp; za0(q}<9ROQSTl``ZkLVuP^r05g=bTz1L(4IRl7yWN(a!Z_p4A<4-e46S%6Z1$Z7Mg zpv|4tX3V!)Z1(Uim6>9-bPg>AIX?^nZ}>p!;0zp}o|A#&Q8qsYrq01}Bky6Uh2MhF zL9yMlm8Lqrck!vWeSo!-dIvyV{}S1HN4R=Z_u2mK@9RB@dR^90*{uCt)|Xw@&J7NP zeRDwgKBu+CS#Z)J52HZ7>Rh$NVXP^ws0*4DIh%dImhY9D?%S5KtW;xBNd&DK37 zM?M;;bX#ZB6XC>5OMt|5LdX@T(>;7aw>PU+{gJ`)nBRcJp=af{c5wPusOGevL)QNC zF#cN9TPNRnpGWo~8&e3dXsyf-|6(O3W{FqPU6DdWlBPo-_;;~#GkR8@v$==HGWNAR zmz0s`yzcH@pawKtN~0<3FbnNx)3eyfNRw>!517w3m(ou9xol~t4|0lT?B?Y0`Z6Fw z{!i47w^(l?oonSLJaP88lM?+F%M7!Bm^;deQ&2a)(O3~q93@Z9eH9^LXp!>c0gETz z=i|qMQFn5#jJ*L69a*@jsZY8xkatv`N2}is*IZECv@YDzKF*}xV6-Vc3t-U(Gi~)6 zX>s>WZRurM=!5(7r|Tdv{$0zXGbxa7;+EJ55t?cf76Dq@#vk8Q(R>qb(M~W2Gnvgd zZJ*b)mzlJdzQXrAL1EDOICz1cuRw}bqt&3vG=w*`8 zOU-Cd%*xld(n)O8;WHa5oTlv2pJ>437p}|9o0ZMG z`Gyn!-IHKMm?L9+@o4-;6Q7=9CeUo46B;wp>4c25 z(%!S&Ua&?%`UyULkxOSKd~Uvn2MN*-^y!aEH}7;xD@k?)YJqnxhsJBQIZz#9UHzJ2 zb1DKfmbGs54`!{IhfjCqe4!QT_kBNYWy!(^JLGNDFSK5-)M^Cw6QxO3RgDqxw8y<2 zp9zplie4!r9T!l={N$NU8j(cF{t5CW_kQ6AmFm+-h0*vZL@>6r5&S>0m}i$7iZ#S) zQ+lV}Tu<85`?$co^g-qyBDwVB40@uTz0NF~_kdm>7gD#m5h~9|B>oTcV~x32j?Yh9 z`l*`idAvPV?==57P4zWc)zo&DLd|oV#Sv#Hfo?4@8#Q{#l3&iyPt{@zd52jCBmty$ zBNxaLT+K+(NHDHNQ!cP7fH9yKwte}Zvaahyu)a`kNXu312;_!Hn-lGV;FU(IY%G!W z#@-?*PHF4u3H29KlEkJ@%@ply)bYN9{d&g3@A*R9UZ0lUi@&1UUO z*7VJtgvkv4J{kw;4 z$Z8tJTqI8Pdd=cc>l}3l0u7Z$XU;@tJ4tWd+Ila`IoX3*OlkWvX(+ZxfI)t9&BxwQT-JQd8VFf_D6LMT z-x3>UTMx%Mb!!Fo7|2y=}l-W{4$ z{h1RRt+WrN#1;)$V|IBYn>_S1m;6dL`D>Un*RNmvSY<5z{qeUdQ(|t{Pt|_DUlB&w zBV78xAaH^L>4Sn#zuL9H?z7{x+Wbh0c3`@c;@UIV$20CbzEKCR9=>J560?$o^sCwJ zFo$1|zrEUS>$6%%|GYyMprAR99aBxdcba~kJ};q(XL4zt2e*W(*zfs#4*{C=%R=+t zS)`F_-@?jbuQU$+l-S~eF*41X| zgxS-P>rf0`iJyU@Qv|M^fj2f1OqPt9rk|>ZQVz4)DHHGwD@4x(1=D6by9W`e< ze;&AlsUT?8Dg)9Rww%9=xAqO&oWquhrFo&4HOEd|0nOxCO=RhYFCAHmtH)BLvflSK z=Q0uMe%j8LjN~~4ac3>XO{Z%1BPaWHrfi?txzZEH`}=~rfb5m@;<*;-(3`9?WP z_Pk9$RcEYr+u{U9!DYR}Wrg;cvt8D;37d5QS=G%`$!b@5hC6ZZ(d3-`9|q={$0BO8 z!~6!U5whLj%TaTB_W(2Y*PcAd=h!Xio#~?xDGiiy+%!Z2GuZaVlzCgj{7Fzjdk}aw zI~l=#7wq5jo4e5mjDuwUN%fDcRnNK$6g!JZB+e~rz6o>Vuxp%k;8^)+VsxpQ={r`j zUOBrP?ASe~<;&)*IdY77X9GhWh?4e04aBI$x#!(5t+6xQdcGABsf@Mmy!&D;KEz4< zChI_(4|mpG8(1Xcb{q7G>OK|7hUH1_QFpbO8g_$+JFV3iVBE_nt2-kC!wKSAiL>^| z{U8+G?2&8J_nLzkX*<)q*g>YOuj-U-RCNq-=9v>!4RWapcxRYfGf(l)w2o88t>Za% zW8e}Dn{%nzr9)5-?&8HAJ?z=Buvt%v=9?G(j^D> zPuzAk6M;rlYbA$J?zF|a%WZy?M$!CF{)FEGY4v-ATJ7^{$@B4nzcduHABhG?$RYE> zH7i506RE-8wScfxrznI@!#6)xlL~?SBkWx0;$F?NrgyYrYEhq< zUJU{g$8a?(o7nPfKMF9$;u4BEnSIP0wTcX!E`gn}U6FWObRg+O6h-E2?CHFQ3mdB& zbmGQ|Viul6LHS9EA#1aoWpgNYn0PtPH95fBC|yjXHq!Yo9#qBbszA;>ptHD*W6310 z1}3gmKdbRcY_Dfd(ZpPBH8E=`BVQgL`uT_?FVGeeYQ2Nn@m%NdB?`brf>wYIuUQ+q z6~anqhm_}qVoQ3cS~tZRbPTm_w9)LQljjQ?vIcvpeMB*M2SIbvdbt&y`W%mxJ^M%O z22kN;CqY`1LvJF=^k4~bA1Cqi;%MTA(&0EdJ_dRw2fh(8%3RGtOS~o&vv!4Xlitbk zlPx%dtXbH&TX=A3)4E8@f+9*by&7p*n4hY&9Km_acS&}9$-y3?iwbzTZ=gFc@s;M? zdiF@xc@ZP|j|jJkxaP0(Ao7J`=iA{wabq}n_C{`rhLa~++ZWpXqH(A20IAc$1Ku-} zS&IU|-W$+0=UUoNAaXwAGP=1#LA znZ-~osHF8$sr7RCKiztnK%-hO=5)3%EtPQ=SkDKb;?q)bD6dD%SSXV)B3Q&T&?M>b3Ioje>bn5 z)ddU%?2a7@DeO~6@J}=uz$~)em3odM zf&6cw5Xl?;hX4jfaUnTU8&$If(%sgt%gyn7SIi9*dA7j{dszj)?jC!-( zX&LQkk+&z-3d@B<5@7n--8a-~Ic+#&E5tY;Pv{QhLE8j!XYm(;^^+WWB7l<5>~_4HD9PF?)cK*S>@CrX zb|PJLrLSvNHd->BMb`4;I4&5v39bmmcMqlkc6sfo5P%4UnTUF+Fp@+6z>ir(UUMyN zk|(M>(_EhMgvm2bd45kGt}*4yQd^%k32h#PC@at%?rFYrA;x3VrK*-vSDpc7@=hfu;vy|=Aa;gbG0 z>!<31~k zQR?V{D}4WjNRU_^4eV@+SV}^)mE^a#Jfn%11clsK+%;Dk&AZyl=1#o}`K>F|x)gN8 zae4F5C@fiCD7L@F=~mZ+p2OoQ$a3N?0f9l`f=`*a_Exn|F>tKt(!9E}yGoLZkIx%< z4y;hK?&il1M_>`^^lyGi?d0JbIccGYCWgL3ZgY{;;0aiakLtSKi^O?kuwhXSQ@$s_ zhm)5q$335WhSr^Tzf4_B(MMe+KcW(OEKlAH(|7)Ka|f#FX#BhM)0VF`qow;yjH`~I z#1He*Ngwl<8aFIZ6>4sn9*wJHdpO<-S?@ASf2jfu(>VjzcYR`ftvQxD>snTS*F5xa zS_ADrwL1)S|9MBJhfg*4f#n+QnMe`lho#iC@i>8KYlTuYvzpKPM8iYf$-;Au7Iwou zf`ClpX-YFW<9#=im&1yNlXofFQ|!e)2WOsuER)=qEY!E?aT2xr7x15Us}m zgsN-sh4xmVEXAJaqqkFZi|r@VshP}c?Cm>le$F^G4?UKe>F~NA$eHX@70s^mIANOj zk{{FO7_T>|dnJzvPd!U^H~yb)grAMK@c>3$OLgDStp{k-N7t6oXNOB3!$u*km9Yg{ z|NHS);YxTGW%V~HQZqkv>tK~^aaiQ^gj$+^8{E}(QgR6O&5?TP%m|v`jo~?Kdh+fS zZ}@#*)L4xpZr4_Po!o-X`J@2b#I|t0{E~+HQfC`pG3GXX%^My(8@`$M!a25I#x;zD zU31}hWjNsgYBBkZ2NUhq=h+*hiS0a}!OP!mNH_W6(38u;@qTl}q@)pZarS7 z92*tW^qf%Np1iOaitP;J9fdN!!swxyVgwFIZRQLDxNuej&uqa?EeF{cnL)40#d=rz z2?l+^$qaf$3?v&y|G}Y|P{rosm=GcIJoBfs6#J6ar(8;XnAGRcvsG#?ygxIUj%3O3 zZccm+DS246CDu`X95tSo&X`CYDkG~qNMW%xn*nIqeH&(P!m_+7Q5+`3!;f?5ykMC_ z=Le4^K8t^wee@JXQ(f6c*CH!M#8_`OA&0iX6fn zD`(|mqAn*(oWF&6=D6K?Ub1II!qUiAuJ(SQh_!cMcai3(lvcTqs_t8cHWYh7bWT!U zH0hkcg_AFqw^1w_f5QHPuVxu~Bq=TwZ{Gq<%qU-^Jo0k)Hfvta%9aozbhDoJ>`M_` z;NoFMlBI!w$=gcBCU(;}|+?2X5XZMtsY=BJHq5 z)`Jy@EkWv*R;9_a5hrlqob{eJbQO^dT$^p#`qn(>&!8k@IFl&nJ+J6khd5YH3HVoQsXR!Ch=5Z@-fxGh3Y+&N^4c67<`ugn3M<~GSBMN{ z5%GlpUFj?-R;dEwYZW)b#l3FhUi2{Jy2}5nPnl_<3;d z*aKZd&LRl4%9_b~;?p>qJ*hBtEDbw(LjLhIKh8g%){#8{wsdE@ZF}hMkP370M8Pq1 zsKO%&J7TpUbsr@xF6H3SJ+|ZB@hE2FwuMKR`jnZfDVd8$BYaAmPs!rZQKSfuPC@Uk z)^qV_51ZCu?f~yC?XkB)3se{QavroafE{Kz6PBVYh(9AId{ItVyOht9o*V?4gFqsC z9+B18w|vPJO+C?dTn=Tnre@(;3r_iRzW$5GKaCFfE9kQ?iW!Na)TmHjuEg^qc;69ap}r$I=J_EM z+sm%y@BH{(b$T()?+CanSa-v*yy%NJu*5I{HM)$)r~=?Gs|@u$m(8I8xQ6lE`mi!J z*gnjIJU+&SM2uTR#3~|8sPB!B5;la_GCj(;mheV0D$hn_x28wKvpssE za5pfFW5aWd@N;hFEt`GA`z@hW+>G~Q!ze&j=sgDx*6MRYSAGotIA6{SdGxJIM^D`E zs}#|>ilD{EXiv5(reM3_@hh3%Tx-QpYrY-M`D3$Z8^L*ZW^mG>*0&&I_F4I?%xhja zjJRtUHAmmfgTuX?H+|MIG2)HN++~%X^jai+;1P zdFg-i9>!!}89mFeS6BGBvYhSnCpNAd94Gi}xp4eQakTtLMp~lGEm;Wu%l9w>c>;4i zCq>9eun3mDQmZhIZaHW33dD+*P5Gf|pR&u;#}_m}6sOH19I>&tYwttvc5#QtC_=PBh`T+i2qV)&`my?`}X79IU`c{NZ(fkp))p(lAi z!MU2|#mgASoHvrJy^a50xb&_oR1553(xF%pn3pl5lm4>M;0KxOOT%S4PB`3`J(L- zz9YNpUI6&K^pNcO4c`;ZU!=D@^LdnPxx^#;kX|Yx|73GkNh3gQ^@SWDE)U)I2&Ee! zvhRfBA3&Vni;OPVnIW1#d~~#siP;#9zZn_uYW;vkt!)z@b~kks+t()|r479PIqg9j z4<{mJ;rtJx`AcweoXtK^IgwekYR+1& zr?$*w)tH|tzTLdZsux>>C6<@CE*G^6T~#>${CpQ9O=xl~3{|4U=D{ySfL5CY(%h4&)@|%}1Ghd$xzAhmiMUp29<=$kkVl)&)Apyg)iwf9 z&XyNV{=U3TuqLZ^9U$_!+Dxzta}Lg-nuESEA6bd6&$MGulDy|~UJ8rOE;3A<`aTD- zLA+{P?{lg0{MO|;!R0wedA?66X;#MdS0QVcSha*; zvgFN6^i$PqjvI`_sm(RMApNT#VBOszCI;%D1?fH1VvFsE=3sXZ%c+*AdSwEF#}f69 zef3lG9>bP~0@!cphr*D&9mewac#zQd6Jn{T@baM>^!xGJf8KU`MccNKQ z7k^zuv$&q+QDpjW)BvGK2q(K!&M*{oI)Iy=$}yU}BA>;P(Qk_ML{Wxl(sIdWc)%i& z%|@0~F4-J?jw72t)-Po9_5TIg+?Txno@|C8o0f;dxh?LiVZrpbFZ|+BGIgjxml?YC zbKC798vruTd{jmsa1c|(>ePa8in!3U%_N?f?04j#t_GgweICg3(uaC}PAg|jon5O8^3SVC8B-u z>f*1}4fw7;e|7zU4_m*P_($PlDQ`s8^Nf$3r)Ovr^LmlWI;@NGzl|pLjo{QdQZujd z6yZg9^bYaP`edz$rR9o)st3H)+BIpv`T^g>J5xWV|B=zJTY~Ah%a0iqEBA0NgYp?0 zlxH2(tv{#5_*?Y@I#at-N&pqJ0IvAErJ)FRj)1NdRx?%^0$S(@sJpM-Zj$AWfUco7 zr?MKzq7nhUbEGGrER|?-lqH}cEjX|Y3$f6ZsIwda{W?QH#k}SaP#W!c8H2ODEeR+$ zlEp%J0vh?$?T9e>R_FA-)z3gnEfqEAe5NjRm|N_5L575$4&*A}Noc)3Eh#z2=G!I- zE&Goo^zW$jMG0y+0I?J7NazD>Ptd>7M`cOqL4+ZpdHk3a`?30`*w*_Tr#zm7_H=o6 zQ=aF@LltF8{te>+iTY=its=#bxyL2vD|x0%Uf`0StReXZlD8(I8j>F$K)nG8y+~ja zd7q`L9p*IO4NpQ3P+mV?SrU4vR+J;5>lPD4zcxxgRr6nPgYhM#E})$M6A7)g{SXOV zcATp1Fgsbgy)6jgm!i{ z|2F@pci2?m-@L=7B;H|LuAk?F#_T%l_8yxd{eOCoO;xd=Ig@kmvCZ0^I{*E9Y-*DC z*!qNL1d zr3UOTNmkydjDNxad=EgHbN|)VMXTb{LLFZzkEDz!n`f>4m7!4R@HgxRaNQx~EW8(! zbJbA_6cF@vaPsDvy?iZfcS@-b9b#g5SdYjrM53U)A^rt4a)O3fS}?Bzij~hK0&>bO z!x5Nv*6`?phuHBPYFEKta?v&loIAoHAHKDmL5revS+ zphe-Iw_OtNb$POPaX8+48M>ML@Z@jDL7tmd=*VIT6QnP8Gyoh=Z9p1x4&HpYS&{sy z2ek@t{u8Zn6Cawg%p#^X%h0sA5QSR84C6gyglB%1?uTZ~Z5Z@w%i8??=1%QbZt?$= zMEO3^#Ia82$Rqc8yyv1VL+VM@DlgY=-Tby^730>=F!{Zz4;e94F z#NVq2BL#oGs^G2}s;1(U*2wg_n%6?Nmn`l zxw&X0SZ&qks0OQBpCN?*M}1ym@vfsjz7*N8ZFoMnRTiKH?!0+WUC)L*7@9B~N6nH6 z=n}ua{IDqId~*f>}p4doJVR!#T^8+NQ^UMPMkqsMU8Y8J}Evjax2 z9u|OuF9<=+lF+Sdv@(-JT2-6*W>>KU1?9FSO%jhKsw~T_TBL2Jnv6XL4rmoKEP1Jk zoMPy_SzVuY9#C4$=tG*k9>%|g#cdDwgc(ftv-}wAEt=0&_;*4vL;1b`RixL@kD1qs$5E0TE3@}_1T_$}4URr@`dUoY54^2A(K)Uzw2Ttu%+tIv(OCSlL zyUC~?kJS{0rQWF2;&Ncvc{HN|Lz5I%W0e6ILNQB3UznRk+E7f&>e>6Kv>*^3bKeR5Ze{_PDO99UsuetsQHyGu z>}%Ec?1No#N*+T=bEYluba}IjHch_Fa9>8>vk%FXIZ9>r^JS*`GIhSp?!Jt^XYZLQ z^RCK#g>Vs09LZgY4%1iZMM_UrJxL;hfxuj>EF>=l0rdSX)!EhRhoGQ5^-pFUycFMH zvB*kj;g=7cbXWAyuTLGRLjtzYV6ViSrIyNKz2(&F0GdYU2`?fS`{{qga)@T^QBO!Pu5s{bA^na0;^70<}s%z4+t0S_7&s(*o zlVg;j*9B-eK4(H6ia-bH3O+y5BkhrjJkhM)&(7^U*)L-zUMx z7Xt~gwmTuyUvUaja2k)b;e4b(PZ6;`d(_8=^^PAyWUs_cTsf}qRp26vMYFgmNPuNM%e1&*I0u_&b;b;t zD#DrHw||8ovAH?iOjhlC2=pWLr0y^%KWc=%IQCdRd;r>5sd8=8u~oiuFdJ(4$Gc{T-Ulv+`J}IdZ-*TCj`i(4OCHfP7Hx-lzMsqZ7hm9g01P@* zoux}KK4|8Uh0WWHvAyr*$22edcVQ} zT|<4dcXK1j)O|`bray7O6Mh*gp}$Gy0H#eH*^_46g5OUKCaYevU}2=Rz=U>EBPHXW zq~b)?WBLKW_ca8`K06;s)B9+(qJ8d9QgY~EHMQ7Q(`P+Vvr{g)v)m>>q~yn4@;ge_ z!9JyzyX3cl-`wVspHcFjF8O(1TFF83-#`7v%G6qc@fIsv{rzC0jOZF9u zN~p9wRe4J%i;P$=Ml7gvry$VBG&3n$#YGlsU*>*a=I_+$vgQ~r-ly7bP>ycI?L@Vk zC2pbCYnWnt(@izNtOS>e=A~a&4Zos8YWOPJ$Q~_O`A-G9=^HB8_Mmg)?C$4$$JMis zi0(TthMmKd6)hqXSn6A>0p_uv7nV{PwkieC0Njju+oZ0tdS3=oxlnvT%oc;mM)O@?D zt4peYD_~`KdhZ+iB-3!#2K7qCP(NTTLPaT{qrrLDvEjtAV!H-xY=|vwdK1}x(mUar_io1AcT+h21AEr% z;htRoj>KP+4CUn)M9|^+pX%!6kh;l(tq_&2YyK7?3W>5SeTP3UM80X+m>-%khhtJ0 zVDtvJH!^_fU;A_hU${{>Z~(n6MY=o!pd#^kI^GykPxtnU4EVbQK}kNfk^JxC%i`~) zx1S!(|5!Tqftrb)eRUWK*abE3PQ>k5Thqt7cdGuzJLBtI)~0t=Ns zVt7#X3oeX^NUrOxbXK80N_JpR4XLSQ)pa(5^2S#4WNO~`4 zB)yl&WF|TMvw|?1zp$lD(>_%dV&KEzOE~o>KBZ5}l?88JiztV>b_-$f+d$-# zFqoAL5e74op%(@tonJ>J z6vU6=9IR)7O!1TTJ&Rmy5i;VLy^#@b?j0}YcE}bo;t8yqaAE?|!~>Rvi&^P4b-0fX zTN$zLIWjS3Gig?T<|&X|GU7}=m^>>Zu9pOqta_#a&S?~rDH-u2;vCDknHb54Q-*9U zBYvW^Kt{Cl<7LE8Aq?{+BT5cdjxQu5K5857CL;=tKm=4i$%w3}2g+C3yfdsYs|*qw zOpkV<8gw!%BTiR7WW*)*cZSQi!>j5g{6EJ{VwzOgNrZVo+?}ynBQ%EZm`NU!ZwtiA z#nt4MKUyz6Ra%ZTVl@8&2c!!Ik#`a%VyYc5u_CPvZW0$wR!1AGLaRjf+e7qfbgtL@C zyhsEOp`qQ(CXZU5+tWe#1uZ%VZA*KqrMrDSmXzE}Ju3K1U*P%f1$I_}ODF)hX^B1H zU1!NmyScy@nbN&TFBSQbZ^P2>Gkv!Y^#$~u1Jq_@36t;I5F4T)*x9%7Z4U?D{@sfV zQyXsrI$(*YsmMom0kxZUU!;T9Qul1om>s(Qw@7V$za^aP^BjWD>^!%!R_w;4C5KKY z%G@sKl6~$`a+lC5S@ohzzF5iQT=ES{KEWkF=#u9Om{VQyIZAGD$rF8PJ0321u#)Rt z@`XN`AF~U|=4G(Lq!X8SX;PWQCKp@X}$HDUPXpXXN z`E~QEj)fVWd`2gbXfmci76bmE9Nm^;lje&UyK*^aDKw+)3|iUBuEuM9t&e9w$aoxI zkj8|dBmKJgfvw~jtzN$mA=bFbezCk6G?rK+I;90ChN~En=61fji8MtN_ae>fD0L8N zMi7HY^AKMEJd!VoG=Jw)lw%bY;91y;G~bvjuU5C5ZGzspAcI61nc=J@qgrMAaW6RQ zW2A|`<4QP*$Ci*NBay~QIfy9cATmj$*^IORQw2j}g&IjT5=wIAjIlH$5l3eS#k9aF ztlA{Ycs7TOKx3g$LxN(I1R6{&qvhrJ#}VVHN>pY<5U5bV%QTe=Bg!QCnJzgd9GbF8 z2nQ>x!$_Sc(kP0rX!%G{>`|?4l3irI^Od9*6|^pawEqWkOQq#YqxqYnJ((Ik5e)kz z!=IB9d)0&!gX9=uz*)8z7-<+@Au$O{y(~F4Cp@0xIbHfY zDOvE0a-nk{5^6nz7~M#lv%1h+_T6SzT@=ASk zf3Y8+6%#Mk0KJ9x9i_Gp`bxty=hH&22=v}JGJ0j5@{_XSq~u=D03XM+s`B!AWGMK^ zfwF?Onr2wJL+}ZXaVfc;$-qDO^78VV?|;d53`-bEAKk|{Hq*)tXRvT?bL66C-F3TX z%6s|aCNf29)>-+Z6n?Ty^2Zn=Gt&_pY3n(3#RJe|$sdr_5L#AI%ImQ~up*Q`?qlKu z(c`-kWw4^h>Er>sN+nzTOj{Ees5%T;yy#I$7A04Izu9iEqo_&8k+BcS8b}=!S=0~u z-_y4#4$K@#PHoAXnSA}~J4fJnP6d{Bkp=`6DzL#f@rEy;?;L^SSrzz=FJL$1`M$sd zzJR`S1dfsP!i=MUx8A=?$zY{=c!V7UD_otTDrx>KU&}(3m!S@Y+4 zXD@O5!|_E2)WC|YVO}M1EF#WH9JevUk~p-nZFwp+jx;qkUW?tCClc7UxZya->X$7} z!DO?a3MxqxZTzZ+jJUCc2pvg1&-`R5ZP~=gv2g1cKv*c>BZ5_{) z9cGGhu&xC2JKu?evdjHKTL<*cw{yGNvG7^ZqHQPI8LV1F7&Yj}dccnI1up1b;5il8 zfdcUGQ`H(p_Ov9W-SqTD26iv z+P9mBe3ACxzhqsYuKhx!f0-L6KDPH^Qky@3|Y%;t(fVT+9pyvP1KZOj&Yh z1&pFOQcx!QG$`2%2S0VmCu(*|T=IcR_QJvOKA9i$4Xv8rfDKg9LwUUTv5(6$OjX$a zC;NO#Yk_RjwSxuVX(j(nRZL^7ooMm&20L03EpC`eFOnrYAElqFlYi+%i)-0(wk2Au z&;SH$kVco($^BoB3 zf3VMd`rp}St_|!ndFPXG#y-=32KJfmQ}&rQw1s_UwC3Yb5{GB9&(tO((tNQD4nt4k zhza{cYkN6}ErpHCnrDWSKhGc37TB*1kAIz#JeZL$7@=GnQ}IEmem;}k4NS78zxI&;nKP`4s|cyrDA9COVNyPIpC%o-Hz&N?N&us+_e97jaRg&$a@#*hWV zWpmnBRxq59U!Um5OAVg0VT!A*wL&O%7w8D)PoR5wsY$5+CGS~-)K3t77=qNJ{04Hg z9lv_OwHUL3GT3eEGP`bcO61?#Zc}{O1EC?@??$|!TD`l zXKGT*oMq(EwodK|agbaBK5nU(E8DVvW1q<>tw5*{2xayXs+pID zZ_~+!a45$(`{#>-$N0@?RVvglw3 z7W7E6D0v82s_%=b2^6!q$|#hEV!CUaBLvSY%z?#hz#pJURjs&-Qp&21wM?ac^_?RP zKSoJ2-xhG^nkw^8-|maPjJ|UO;)hjcx-S#3o=^EQQ+*kI=g7qM^v;~8GCCC4Y$hw& z%f$QI!Ldy8F)oKW-q&&|+3bo`-%~~4G$izYYl=C2n{x4gVTZX%3z;?Y-`QcdJHIhG zXK(Tg-I9i>-{R~od))Tl8DipRuV8@&&x*Lbg5xYP<$EK7DJE@m*@L|jt|QzM?L7@j?KoR4<)nD5>X@+0}S(W3rLeTU}JMSxTGtj*hFrUoGS ze$DK4x5vDLd6eXP4978BO1{@Ajg4i27SBxT5C3oNG2IaS-U24!cP)hj;TI9{-`Hc` z{Wx?Q+Vwx#WB&S2qEsDbAxe7j+@jb{_?;o_Bu4GIm|GMB3*JVIxyu&#KeNX?kh2`E z0W=$jUK)o0S$P;i7=h#V9uSTHV2}BQT}Rjh7|mF_*@G(pfl$1;3+UB9@`&2JY!85( zP&^lctNgQlxldA*^Y)l)e7R}1oVCZilk9;|JRwWRE_XG{9`kDD&}ItIuY4(?*iyQa zeA}n_c5YNVUTFWbZ|6Yrh@4rGXn+T7KVKk{Q{Z7=U;}wTrKi*y{oBsclyQa zIYsXCMQnd_X!j?+`q{pWz5}?X1^rhu-X8O}c15(CU->lTQ3CJDXb6&hZUm15d(0{5OpyE^2Lf}9x|-~Bu98!r4^{rm zC5Hsbo-TQak~tbwa@Zx;sr2_2h(3EOnNvh15AdaxywW9a&L^2OR3-1=lld_(yX23Q z?Cmkr-zwM8nJWDd$>t<=oOh0uXKz+Cvx5pg+86_MiFmyVEC0Viz zJtIwrCOh51(U3!p%;8Lx7wj!nIgaB^#a~Z)P-l)k=2|92t9ZPH+Ebz(eVJjIGF#eX zrnWk(d`@GUC2s$tJ!UT&RKp*!OtYq3p?x|35}3j3Z#X^Aam8I>Uw^C2YuwAM25>fu z*!^0riJK}#PGhwZp@J!&|hUD^QG&!6{KEL6_Fa9q$cAisB)K!K%*GG~C9PCwV zpn36Pw{V=tC1|gRygKo_8BM`^0Dif}_45(8a=&^xG`GL-J?1GCQ$SWroK;Nr`K*^4Og8P?~uL;7u{(d zGT}&~fP?Uc3e$^qQ!{z*|U57Jk&1l(cj4TxcrXlH8j|!>v$*R z*luXcqlKpZqs?8zVG8X#C<`K;owg<3PGNlCwY}om51!fO3A!cW#L2jyOod|WY;X5O zfYci*Cs#6o!tvpUuT+z2u7W;B61zr|<4e772>ixGgjxV*BC=p5H2z6hT~qUj(xTjM2FVogC~`> zWqw7DK7YCZ3=ck?=L|WYqjMPPUJEkl%$_T^&zs(>v>Wlsh&`VA2a+MEp>uOQ(0~!E zP2xj=Xa+5fAS@OI1Drb5&p)m_t~!A`J;O>9{-Z^a%5Jy;Neew& zc<v_bqv`;HP9(kKc#XFdA*Igc(nBE->H# z2;{FVqNMbvZS1Adcm*-VM4+ClE!A&(Z5bMp7B6VA{( z%nBhKqx?IIS09;yECJt8_LjknT=LsAIUS28jyAusB|B3xG|~4j{Wmr-+&|+Z{g8j} z)cMDJ$r2S$nJ0fam4QLQ(z_eR@PwMNcSjj>`O-~{E5ZXz^RWkqX0%1}7iyLkvncFS z-JN=RHo9Un&u51_uchN0dk$G5C-Sjn)UEP(qmSVJB>~0In!@WLejPS9Uf+D&!8lxR zW1zL9rJ=PHE4$~`A~4=W{_!+VW?B#D!?N+@;>k3x(Xh{aFEp=zp!O}@l+_WWw(VKc>`84+k2Oyy@6G#VgWA%+HgiEmje$DN05%oKBqN%<*pZQP zPHr^qhYV!$l9x-r^a-oseB^s=hIfIX^8PYn_Wpn+(Gs<}_yuqR%ZbTGU>*6l)aXOYpS7p5$o@z0W`?9p1*$*v!Ia`Zd+tD)n zl&{>@Rrk;g?-P!zUeG8kg#!BLGSSl79Gr@kSCAFns+A5EPN09eC)VLm{YHt&&GgdD zw4FYqe3Wl|PFNSiQDck;<|Q@#F%;QA7ir>T3LriN8nxv^2g&3JteZWA5zq9HD8y zMwcgV%x!8JSzj7lK{B&SA7}OXNUcyWl{drVLy4aM<|xJE3821*~3)r zTET%1$MCB^%^loFb#3waq`=9Eyp@Ak;t9 zT7~XCl01yCQy9!q@d2t~hAk)b8n4_cDBp6c?9Rq|f5Jwo{+T4{PyU<6 zn>F^>*s1=_$9yoEq7vJk27t-J&edd0R;|?!!Y++JYKwX&&W!0SeN(M|^`~JdH9uSC zF=){KL;;=6Cgc}%u1s*|1{~we7~3>}0tzsjvn*ULE4IjX5LEG%uszmWHkl9`7c!;Z z3wr&#G?~H3Pb666zpr&Vqg?I)wO)m+%CZVL&C|-iO`Mx;iy=MQHW+&ST5O!@Qv{+A z>Zwf16q~}uCjqyXXfs2{+|;ir(BpW8<|6Wl)ZR!6D0wAr2JrbMBR7k4e+4Q~>ZVYu z-W#_W_&gXnX1lz+Z-)J(wrQu(bHgqO4=yMV#a`Be*8ATHRGb62jIgoLCQSK(%;5xV z|D08Y-23l=G=hH&A}u=5gl4x|WNj9!NPowL2f8I;4V3(qR2D7z_`X8y<{vn%0L!8{Gma*l6)Y zOQ3a{jrOy$lid=$R<*`?z6-^!wj_g{tz`jZ)itIkETW0a%Az%wRfJ+{UQeoWzbHIw zmF-v8q~QZTkJKy--MRxUz`liIvp7!|M^Us~b8}NE490M|UA~d}hY$E{&{D}m@YB8f zVUk#^&>8(;tBtpZAEBT<^%3qq_faWd=EzXIX0v* z@nJZ995$XVHb%u(ga;3O{#v~0#eT{eKgyr_MmR{lJg@=+VjlTBBaAB@O6ay_>q4FA z4K8e_1g@qMM>HRMVW{W-^-@eXuLICk%_H~5F#$Utro58 z$jgvK>Gs*Kg^`m=7GX29tY?Hkg>u=OvJ|)mp@gwpC-X&;X#qzRVS#Tj7i4uRofP}(HYEdE<0xUQi}Uph{={;cBMsH-*$Op9JA9@qr7BXGtgCF^)))9oRry+n)q zv3-RBi%eR~(9<8=MR)tzB{JA$5+Z;!vXS>1*b?)_-4^vIQAX+Gs^)hR%?SoTO?p@L z-z3aR*8+aj^yidec&YdMYB80yS-maI`QrjAQH77{ze+oRSUFqYu^^5`!$W&1QqzdL zO**)X18yZMD!FQ+vZ?d53t^hFCFkpokuSODHF@T;om)DNRa(V(59347rPiUQGI-)Gk)brbh3Oh`b@4J@qYvdr&YI=xMwV zL8zB6PE7ik0n7OVj=L4YZ@ZJdSr-z=5*;qR@+IRacZ#VYgi-qJ{+u>r52sX~w`TEp zeCU28U-Zvf|w$7mVr z2k1og0kff<-%OHwmcBXdLgpERUpW_Um^JV7{wG~C_w)Xm7S37Kon_YQwe8iZr99m9 zb@iO6drh||l9?XAvt*6~4wze$a4Sw)v!!;XH(RD2%pKp zHd^ng%v6=q+GZE_p|!JVjqy3`ylzv5+dw^UO3!$p`YGJ@EG4<;F=w(2u^xdn=oUQn zU1RmhFpAYpd!6^KwM^&lCpLPCubT9pB`Eaj0Sg3DV$$OXXrUr8=_hE@>MOok33U^b zcJ%>uATg)oh892?|lK2M1(dq3$*1mxFm<_;pK;Rzvt zy*H*N^UH2yF(L|iwV{3xoIae}@k&Zk3(OXaW`8c%(=f13Uh*ZHS8LM?g;mgA>3c&F z!R)2qMe6s@U>B>)#7}AguE+6KlPt~d^;?AZsSnXuVUoyfI@q=G$f?9h5-Tus)o`tL zr`wtCV2{%?-HPO9;LcSqXR(o4N2IKmL5$e&7t5iP5lvoH)o`PExAywY%%}4Q%{5cE zm$X4!PRJ$_lP46~7x%Fp%NOu(Iwgo6%t{VY-0n^N&{u?cu}PPF``2vdQj2$jk#&}$)k$CBkqZtE){B0IB|jY74WEj-0BIpNBOSGpRtq#Cr- zQW^HZj~~4U@x0$STL*AT`fGMRtAY(wqd!s~KnQy|xK7V~`iwxv;)_0B-34CT>!Hcv z(vV|9G>IIc|Czd;(QKx#v41L4w`p`wUieYKf~c$A@8JbgntU%pqBaixR~@)YwCo5} zxiy^H^vp&|$i2Fq3^lHjE{HAy88N;Au~buJJwBhF(9fksERO9-HB;N^rZ}rG4(j^B zyZG66w3sk|gTh5n_`P_X2{_{V6SvIbhH2IwPbqlXenEX-?|;{Rm3c*OCpE{j^!Ol{ zt)Y5Plc+*H$jg*nRpTMXAZB#SyL?qS&t7yU4dwU&y0ihh;mMu*ggWDG$k*v~6E-B~ zWo2qn)4I(Pzx@rBHmyU}9#m9NFxX4lAM=@-YpNnKsRsdXWp@13)4%CUvi}MgulUmE zKg+(oU|uQI%!&oFN!z!Er5q=(^5yMC53|@FRe5>f89Z>i=(>mL3vWTzylN;1rQ+bDK4g2@X~%yXoybSbPg(vsnt>W0O+&s> z;lQ@&7oSL$dFwEna{xIT2Oo{1gdZ+r@@$5dc+Z~i(#^Vy4KlwXEU~pjVWtS!P4?$T za+!%%c`ye@0HdVR2sh$K8heU?Xw%5p#eu?HXM_stJ%I`Cc3VkW?QYU9*;TW{*a_v;nWvnueXCQuZKtY7&S2vH;Hj)7BldH6b&2zjw@k-rH3# zuuT{80M}PkHGk0Xqy}}rM~kV_?X(y_{6pp-PTBM`(fl|huqkM|?3pwCth41R;-WEj zsGXwjy+z;Z@Mg#4R-Dg?mY*_+B^&xhf3u+t{?#YR6G^=fU0;cYAhcy)G9KD!T~RUx zF-)|BQqXErj#8fA^HKe}SSxlq6!wQy*s zuf`QEPNRkB=8i7%?Ig*Q1KrSh>VlTY63xe}=B9Hv2HY}NIDtuzSqMD$6UfRTKW^Pw za4%~*(PT?2z#6GeeI9KOW18GRJ}SIv9A-9NhnUW1Hety#WZ~s+d8COS-O}@=0vJXx zG9E7tq2C-V(haQ>_SF|vGmBsSsha8f%@iK-oDa?*=&rJAyd%x3>2wFEnlNmb-Cgs# zNMyT~dd>Ch>Q#lg&7Oe0Hkc#Oe!D84sQeo>ahgDwQY73oSJ3R*yUorFDbqsAjBLsg zN+GYh6k8bbHXLU@@nA5R^K3VA#%@e$y)ts-6m-@{`;Yfyd4ygP9WF=f841Aa7}^D? ztdI0_eauI_pYYm9v@0YWdmp`7J&pd#ra4RK$l-4PH?z1~+NK^)Et#VJFi}uNVx8mX z_!1u}%s#%@PU3OmWckuWa|y#h9+P$}mn5BA5dAlkHb2agnRqgFwV#NX6K4f5v8;*E zXEw>=r#sw0D7@uVhIR08Vg{!NxjO{IM(Ok_FVictF;N|l z9(U`x$_$3jE*TX>_9%BVhwdb@N9B*0X;h^@xu-_y!;9>XS+W$pBK?B*mq>B15r`&! zkOdG$N3`b7b8TD%K^IJhr~%ocvhCK}vR*5w;z7Ar;TX|0dLQZkCfNd*xS?2dh;Cn{CAagqt#iOlz9N50`@Gx3T>HbsfZKh-R9M_< ztDJ3hUJdZpPUUQC?J|W0LFH_N<7~q2bB#PiU(2zW7?L!fp*P~UDenVt7Q_|(LpmKt zuV$8|u~c0?ye`qwh<%`uc$ck_*_L$l4IVn)NZNjT^I#7@eqUB$Fv;}l52pdv3_Sq} z+YTss`Vdc0y0MWsz-jncyV#Rjo4|*`1#18zs&$aDKeH?tYU$?fRR%*$>ZIhG+1 ziyfq-nTIMI4_!)a;QTXZ-_@&T7&_fQr1}BzTgJvUG z6oM-W3Wz1B*XZveWg4rsCUvjimGp;AW%&bd8!>KeAD;yeY_LgqcYF~kr0wx2Rj==9 ztuCb9d{3(|KM-fR4)wFk=8b3_Gh(;1SX7W7OvsQ`cw8Z!`Mo9tQyYT3fFA)ddm%mwpzZ{{JN+|WZg8nE>!2)WO(Z1v5ty6^;KaW;UpYw zS_fgW65SZ}^s_oUW&ZJ z=E_~6+?hUKtFV7|chy5mx!c6MH*AFb+rdq&Fvx$<3)%b+S^ij!10l0kZv| zAw$7z^$Ghi+CsC}lsIDQ%fjE_d)l-H@18MLIC$sR$lcvn{VhlB-9mbec;i{G0e4&1 zIiC~nF~~as%GLttr%s2N`i~5YvA7L>MpHXILdrCwU;f(MDueP7ubz<*IaReJq!9F?IRoi9^cFn>e<7#(p@8dVsNj7%%SplU&y=;)*1rw zK%jA}A@D?%A*J0qt%cF*5VIjV_6aG|0-|PQ5!EN8G=`McY|2*}P}Dyrqy#b~TJ8n% zNdI}k_~|UcZh~>vX=3_5M}9#J_Xi&11>?W5t$fzSxvUwV^(Q{-DIx1*vTo4-OC{qz zco5?8tM#L3mmD5DR>k>ujXwr&9AA5c{HL4D`?W!d)lU1Rod!p25Bl?f>uc51w($hnXTTP@l{$Fm�KmG# z$QRv{#{n}u#g4w8HvRbQH%6U`r-^;;{9Gxz^=`!o5ly_#zo-;;D_DaNGNx`M)iFw;?#K|_2!M#$-ZLRjP#$XiZ09}v7S$k zJ9t$L`b{}j-e`3b@HR#I&tvf^;0a{7_GH82?w=VJPdma7oHVe;jnu`B>_yLf`Kxo~ zkM!kNeQM=*4doBZm5*uzPezYqF*7gfllQ%A-uK&bi-F3k?gLq`viN8I7eSD{|Ju$3 zG`+vUxeNE+VLY?PM0rAmU3%z}+QNS5pd?$^v7hUAXR5BXy!83Z*gvm7T2H68JYK)8 zA}GeWW^G$vTL0GAR$2PF(-lm0m~*l5xy*|dtp3uZowukwsd9AX$(5&cwtIhRGJPbD z=SprITf#b3QNsf@>D{XjJg1=HAJys6EV!_n#SFmWN-hcTyJ6=`C>shdUFfP50#tzq%P$R-N9rro1@Oe3iyCetXb1oa$~71bRZ3eoZKH zw|yeObZ1y^PD>=qX{~;X>h$Dx%T=<3TB_5RuoHd%673X8sAQ2F3^7#(^WsHUN}#&w zpX(^mt#VCEE5AklH5S*fRJ>@Hvn_H-II23`z&^ao2UTCOXgc}aXH3be(-+|+T~ptO z`HgMfFCSi1&-o3mmR{5FVRd@qD8FlGF+57B!iOFrq#^omexn~UxQ(wgd-zDFl-Q5Q zM0G|-o2O%TAMuF{Jjh*CoA#twe=9Mo2d?vo?3~p zFVrB%0oGzeDThgPM?)P|V5nASuKx)1acwxS1N#{%8B|c24pTen>)Vw>3{j&!UV~H{ zCuj~@)-k6#^?-_Z%y;{PJ^B5+j7K|ElDAivDh9UkWVkyvDa_7ue|}a_z}O4i@_&mH z&KXpzt(4EGMI1f7^`JNy=}}Uvy{mAzdBOSO{^q+Ssa98ZzUY@~|k_OwW?3Cn#UlWIE{ptP$E7H5wg?xfxTW-5rrpqd8F#a5jy z7PW8}PJw$IMO2$Uhc&Jb5-lvn8D8>Wss<(v?Kc_P$@GnACx#WzeqFmkr`hnfUZ+2} z%K)qFd_LkTQ(H@7k*Y9S%xmtcldAGF`Xj%t$8m@%dn~4*m z4k{^VU_X044jbUt!e1YbLdcX|3a!KxT@V_L1OQjef9mKchC#<{aR#gO5u`T-f+UWb z#oECAGjc=+lAyWM00mA}??~B60W7$nb9nq$u{{-gv}a-W1;mOjQB8u?)BW8*z`NI#Iu}D-+)GcR_CA8P?M?aa#n{8g*7}K^go={sUD`Y zIxFF&QHudmXLUN==~~f=&+2^Z?hrjWl_B+=F@K;=33dW|b$VxEmvdH|0Ax*Sw`~Bo z8c9*%mM~bNBY@V{!SnIaP}vn1^ZYTM6~cv=f|5?95*{oW(I!U1&Tb6+j4tR~o!Ysh z)eR9ynZfry$xqSdzakUe_$M!&? zS)1u0LUq&p-fnZUA$yhSnv;34Wb0tht*ptM*4sS=SFWeb#!GkAq}p>^IOgSs1Q@dI zGw(2s0`W4Z8Y*I(N32SQ{SaU?DTHwDH)(tVIku907)qo{mL@aG(ldI7Lav7x>=fZz zHpK4*e`fGCk{?fC;RWXM&NbMfW1p-jD0G`?U@@M%sVnsIc(_r$Js)CNou21I##K=VXGOEv9Un?*eL$r)p_HG}J$Q%e!{S@uelr4!WcqB6R}gZJ zsPu|q0^JbA)EiwIRJFc;6~43cGYy7h7kdyEF8pFxw}Q^rMkApXZNy?vD1=F^)0B6D zdc5F#&)32VTV$JSVC+%`j@I%xMC87RmhcBTnnvD>;rgT@XJk-<;%zSU>oI2$?}~z1~DEah~b!&?$Kj6 zjrl~&t^BVzEP?sSJhroW%d!2~E4m}#Ml}?*d5p%QEn^=(55%VnuK{D;EX|ZPEl^#9 zaK2ZHPDW5PWU+T&zy4RnnGMZxRG`%zDyg3JJ7kAybPBBiJvW6%$W|!5)*VVs9k07F zS^393H3A`5Z;R@M$6iNv<&YQTA5<=KpE7mV`A8$#lKSn1ou)5~F4{&BP4kO%Bz~9{ z4LUxC^S_x7=Pz09`N;DN%lScmj?*bX3}M-`oJGKv&%0J(RFXrw>1nc}Yf`7{se8uJ z5N=o;n{>BGNHi66lLLDjcRu4qymb;;tDD$)WqOp=$mt?SYVH+p@!R80{_EaQSDn#f zj9y7rF6JTo&#Nwf1H7@7gf(Y6zW_#Un4nE#Bs8pa+l1<0q=M?EJxeh0Sc5S-kIJJx z>m};{xGrR`fYp3^4XudAQ|{wX&b~-e2HB^7)z9D?h;lo_eA)de74&@M7&|4e^L*9m z+MTVu(f+CrsZk0{gKq~WsQ;LxJj~f&>*b$d_qe^pgCW$+K z4ts&1;|H;@V@Z~(+%L3-M6oL+-hiD3v&Txex}7#sx2&JA#C;Bk9goNQ*Uqc!Rs$}R z=aPu!oF`oZ+Vl9v@5s@q(PjR8a5DWH&g}d=(VQk@W$J0SA08%2og_=OswQA@51wjLH^BjCB zyeF(}Pm8HDkHu~lbXUw*m-Ez`KX=b8qIbu_+>li-GQJn}LkG6C0vLN9MQg!dKIg)| z))~dt=3*~eRo{2uvW{i)hi$h|4^YH>sr_lY>8Mh5*xir?-gnD{n(GO<(3o6raCXTU z&grAROun6Kt9nTDwH7>uU`^(99s2zQwricE4y|-AN%&B*0#$UM#o4)$fj296SU5|q zf+v-^W12Gi_hiZcNG^Xj@{7@4?of_iPqZ=Uc7u+z^qU#_8GJdJ-QQuim_@@g7@CEq z!w#)XI2>ttlcK#})4vsnhApi@J;tdErnhd$R3g zY<1#X9su%-@kHB&V~g|G?!~)_0e-iGh_CN)Z$`WpG`-0R)p9;6*FcYM4Sf?c zhm^Lt?i?=Reha+8q1ueU7_IYYMxpPss@EV)%p5`Q@RsT)i+dXZj@(I4;1pe(O(AfBSJa_Om&o<)XAb~>xTX&EN$)kS z2kOJ)X@hrx2mO+=;lp8 zlBd(pI8jtTtLDJ5y&FEM&P=Ftm0Po=!)oo$$p<4rqGhs7l0d9ALnM2DWidyY;CqQS zqlE9I=Wq4`AAsEDmi#b2F&*ekqj}lD?=(udUtwa4MhbWv<#*HhTsxQl;DspwnM{xC zXW<@kn3I;7VGH0?Gp<}cPZCZna9eU_#&4Kv;EfQ@tCpW#Qrz&46j#z+&MGA*E)Kxt z+Y=z8*8x3Ge{)w1`g=(|Y;hR6+vCOS@A`YCyFWoLqPf*1QgRXJ*6L4jJMJfp7#X!W zcb#777-I_>U8srQ8(;62o}f=8n&0lOPRSKs|AY03`bJJ4XpWb|marl@X1`bsAh3}r z_mhu>z^OjoeAdB;nsddQq3ax=h)kZ8Xi zi5fcYP0U>((c-@oi7lEzCo=(w->9ziQP{zl2zUcQ<1uW0ET3D?&{Y&ul#4mNqVx)n z%Y=EgbF^QKLA>$8 zEcubij>DD-nh0ikRbp4h*75TYJ$g*5%ii_LOTGPR8iqtUT8I zv3FF+nr5uGvi!_}XYz`5VWcC9PY`d9t3Sbgv$sl`a1gC0 zY>c;AD>B}mkd+O$Yfha{R;# zhgi)1(pnwir9+8z85s*dbE`s+x#D1qoh9`H)DfipAUEw}I!E*fNr<0_qx%_0qg=?_ zfZywp5`$nv+DZ_-%ze9R(>LHRy;ut1RwmQy5mrIib?820mv6KK?2^?N zUHEz}xa-2#CxO2_KKz_rfKgI7&3FX2V8wPuk1~Q{ehc+^zMkEszK;^kJU5VE-yNzC z=TE1*5lxm%&nOo0QuCyN+H2D3ehRK#gF|Gg?to(i?j~UXo~z5B()#)#a$|3;2oT3R ziR6X0TuC(eN*1Zz^D8v7WxP|}z-2NaYtrYjki3RF@QOho7(B0EV(UYD*JilVEVZnf z3lCCDG~Y_i5m%NkYWTb+y=hVi-war4fuIlejk(`wYFRzU1GZdBmYP&Ix9B=JwCS!= zGUB%EsBApFq;xeK>>9S|*e}L6^LoyK)$gi%hX9=u9mp==K&Cak<#wB@8Z1l0>$YUM z+gHNN2BK}A%6QR+GS5l3XsBQa!uANtMq~L;^C>TXuU6D;{1@PH_x>K9zR#R0KF23W znP@Yp8u-A+?kO@4wb(|%D@5#{j|i^$AyvqzmX-AT{-PYD*sj`l5EMl(kpm)iwAZAX zl-}{SJ4WDUdfjnvZjyBTZIsLmK0%&|mfz*(A=|{Wj^pBwU1W2c+qPDCW8QLyR7N~p zN3pJw)kcSffS`8=TR^7N{`#We*%b6^iSi&kq82)SAGh;d{P8GE-mLEX^$fd{OkSy) z^skC4*EE;N$y3p=t19f!5i>0#Pqn?k=D9(WHO(&?U3t>HR`=>3Xz+z>eM&&*krw@! zKDVkxocyV;RJ|rq?#q>`(6};vWKnPmCYm!ei36-#{piNiO21*ZyKcSEev^gu{lv%Y z|7~ic22Bvo?#0x5VM9l;?bN{(CV^2-Q3rPz@fGbUX7e|jfe5lUWz*a0E}xs z#Xn6Gio0L*M^47G3^%m5u+Y^0ans`NQ-+pW?vOJx_&BKPPIwJRZxNZ?l50eVK3D7` z?_$}P`a!Ix3uov{`X{xiyBhl|XDx>N-;No$rt^A@Cb?)+i`+{1hk-2o2|tzj{uuBy zE^nfVDKfB4KD$@<_CC7~@SVhGmu#9?P*7jz9)|JcwcAmA%Bnh7E!CgEf&$DFy8*+h z&v<-ySs$sK4Upx6kD{;Q1&rS7-DF_8V~YvnAzBBMN_)^b_%ZDaLI?8F-~|VH#D>UiyxAyI`zJLoqRABC|qtOR=O>O|MlR! zzLhxouo_y3ppdy^7kzYmzJAuT$*S(Y116N#TbCi->_!sZ@rSsqEpMulIU&aG7&JMi zi2sUT?qcWCSbV$>nfVrIr z36~0-@Vyf=D~{JPMe&p#B{)Yk_ig!^MsU6Z4Ff0NtrfepwBUM<ML=|ru#Iql{r zO1CM7Kjj8O5?dPeBa=ARcZr!*$0r^bebTVR0~1f?hOSkcPOnHb{|w(-*G*UAoVVGDvBDh)qv+Wsfg^8W zo9P_X?IQ*i4nw?Kb8B!cpjIouQFku$LbuMrjZ(RemG6f4WfN^(lUkG*(4)jXEHWp~ zV=24aYqH;_QLhk~*xI+D>(bOMIVIIqdxEiiUSiU=3N+0-Bsr&}XAIn*sjMb_yPY|y zVQQ~}c!%%~s^1#i1D4g6KUcq74Y!u27QrP+@oN7XHj}w47{YcO4!=YHn)G%h)tUYK zrk)u1xR#+t_;vG+-oYMhnuCU1)vbO|t?vFvwI#V}czJvEkLM&eTa;Y8B)MzP+Lp(; zSqGm@&!#U9X?V68t*?73sg80Di*;XRF+g#Qs6Riow1HvsqPy!V4a8S7CT~syjon)#<$(pL;@v8ahV6#6F)ikI;4%Of# z&IE_*h(TS3)dfGYafaT$&uA<6Q=6K|cvrN->Vhtt6loI8JvSqiyN^?Bs=@~CR1Y3` z1uf?(uz8xkJcNCBd}M}ni{W3ya#d$%YH8&oWGhH6Z%h8TXU%~nC5fgtB_y;{OJioh zOMb|-Cno6_7B+<4!PjkGe80omACgiGR5A*bPk0k5SH~;hKu$NqJa>7z;~OI}UYS)7eax*s;z9h?XN> zF~H}CIXR*8M4`$s5@xSTa0(FqPUi}K|S7K6Wk375aV%>w6#@ggkUs7_D zx_Rh}e7$hY4|tg?J?!0n17FSZYRt=7o_#{{J`6a2k?inw8l1+w;vXFbP0T#3U+USW zH@h{h#Jl|+96gPU#?e3JBgfIHrL$lJIJsTUkS%&9*Dg!$+P9*)Gnp8Mo26%U)BJA6 z;4M2dr4;4t#j?h zkEB(1H!N+8K9rD@6Dlq>oUBqy`@qx#JKdlr@-Y0St3@f;0`L}?>7d>;P=0zzabog` z+=nQDlx?5}kn6nC7i^6SE;rT-e(AVIikOt}eZa#!)rS#jDyTpA{H@f)@KS_ZtUydB zI9KB+@j&g#Itak_eU_kD7)>;7185?gft~8X{a^lDLz}8J9x)yEUoWw=VCyfGYH- zJVklmy~8*DejwBBBF!(6tvWNT1WoCBCNn!NuyP=440+8K(WY0zGgO~MpF5V}39}+p ztC&VMy+_e?Ju92q`PBo++;dv~LVa(}Q6!p*`|0_``b`#gFBr>Rn)Ur2tWY(J+7RNM z!&u#3&ThFS$na#h<+08KRZ>;M7-jwWr3=}v5v8svN!`*?qPi4gP0S;d#Mp#OrdV>Y zR*<5~jzPhF0M;Fj^YcrDub6t(mM*E8X7x|C_TUNM+|^cI$7CBxP7Lrv-8D?p$Yj6q z-B$`TfrG%=D67P4V$q_MF#n7$gY*%;E5bKflJ=>K^PBAmY@9_^>CY8~#*!ItBIy3@ z{_fNkTJB1wNB>U|!b$~LX+%MyO z9B%uU6_-;_2e29xoh09Qt2=NKX%f}LP^0U~*U$O{`5I$n&Cq_V5Z)jaaA4uZKjJRA zmNbEAE}J>L-_v$Gn&-frX|3Qutf0SSW;hUf+a_8>F|!{tLpz(7At~n>r<&EAnDm&I zSn>d#L^!&OMp~1ZAYG6E7lVTCMg0iEj}eNE9U zlW|!4B>VsLk~%GF>spE>;7GSnwDYT-buE5L5&tsdNrPPL(5NeJz|e+^G4ztT)~7sD z#9UB_RgCL+PPC2(zg?80c>Ng*9mb+kG&xHz*10>q)7ty$VL8jn4u-n%cOYZ(X8koffe)T;?QGUpBpoMEka< z<=^hp`d1JW{)x4$T=BwM`wLpB85u5a75=92n;77kK*nBg!dceQwWW3g6c7wp~gx|n9 za6{Hwd1o66L78b3$Y&)Bt8wD>7aJaK+@GicN$L%UMQ#+~XkXc0!M4Wyt{+^J`bvJ` z%%vmT0aoDBq}y8`Y5&|)p(~hZA>HZE>4Z-%Bh)l+Tuu4g4UfxiZ#iC4=_LMBJt5_v z9q%MFR|j^lDd`{G2xLl6<5P|YQ#=l`gX#?fKl{!>9cW6-)OF-uJq-hOB7?30r;MoJ z*S#v6)*VuRb|qI9Q>AhYOU)7Py)in+jM1gZUAHm`%Yf)9+tO}}E&nJn={D3rP3lu7 z*M-SVn07T?sTY1MGWwi)QE%Jzm3qtBW5vQ;m}$69Irh=AImUWA ztOE3Rr`Lp!wN2CY`lVyMa@oxX`r(or zT_sbB9@M_akF4nxZh)5RR9N2UbgA1xVHC2za;8tWo8+yq9h05onPt$i5z?foS8T^* zr+e@s#okx4MD=C*{QYL-XV$#dJxhKuq~%9|jj?jWSR1jui**xxvTq2SgyjA{`N)uL za8kNJ$yV@ykW#7`QYu2huqU#gCDWncPwJ@70>P~k#dti{0CJKUIY6@Iu(unIjJEqC z1@#vf`b4%aG#cmh@asgk&`Kq9U#yd1oYmXr60)Uqr_INI5;|76I!V+_p8@#DdCyZ0 zk7q7WRZ~kw2|&h`5TPcH%|}dt=~RR15hIUozbFOH!$X{5h&bI#`g4n;civr6CZ@Bl z@IzNl#D_j`*`xh|ToBY;EaFsbT5}+_KdmR4K7tVkE_;w~WvF5Pl%x;QwFr=%>A-_b z!x%Ys4KnE@ZExMf!|N_rWw;!90i04b|;HZi#;o~h&}7_CD2+vlr%;6GgAh3MNwVvgNEwEYT5CZ<(@lZ}2QO{!$vE zv>?iL!rU^0+#eoSH{hXU+IKH8a}`ODVzKU-@s)w zK=9HKJja55nlIVkpJoFMg-qSqGsvPard|=+l|g@YubrSHt~A8yTeiG<}-s|S!7n^9~4nQU2uO< zK-yh$`8jKPLxwx(9A(NAQ!rf8M3)%uWK{u*jDG1rj$0%2+(Um|}s4RAKnID3p* zJ?lFfT7LG14XvR6o{vAz=!>&cg2fNJS6~Tu&bc1f@2Z!9>TJ*w$y<6!rZWmE+yj(G z#o$#IT{oHP$vsCBoSeq5JpzM1g|c;%>(yEx*9+SSEop{C`sdcC`7AHGl@EEi2D#A& z7t24wJmhTGFLUYU7Kbd~8Tg}1`O-r3T%iTp89|_E%UIy^Clt3oM@gJeobO)SUVPD~ z;T~c#W&bGB68TE^0ssYl5O^LftL^aKO&DqBuC5jf22mQhTr*&0GR1>{e-dJ8*VtE_ z^b0hqWQ+2}%%pNcs~r8tjQx+u)W1t|&eyE+`g3P$sT(FzggeOsUci}$spEXV4~}&i z10+4{4czfqdBO}JvZp6EYQ>;l1OyS8nMd7ecIsfRa9`V*ewE{%UX-OpjcF5Q@5w8# zViKW$~eDS_jUQ-Oe{F;`hs(m!ZBzqh9bx`;9reUFDw01wr`V@=Mz)4%^-xvm&s`cCOCkZR9h{M+?s<)Dz0SFo zl_HOp6dpQUkb67|a7|y~f*ej2Oex0ImVcILIaFpsmf|xL zzkU~f16igbGG)6Ow|BbbDf;Rin$6cqx~DN$G%#hPy?_eI4Uo5x_L2$HqvnR;BT(g5nQ;;G=S zo}-dbflFp9$%pAUfX>mfS%z*)6rfpqX4X}w=0rc1PmnfjTw#Wk#vJSl%4uxqDw1ts zuVp?1t~$MU^a#IhJYz080B}^+h611%;Zt*ES&d&t`e(Y}7ix;}G~(j%b{oE3f&P&w z{9uq$qudd~aaww}P<$*ht-n4QwR3qqJbK?X@aX=YM+?9=Ji0Ye?rT`yy$GWklz0!v z;Yv(qc&n2`E{mX>xN%tbb@tJ;5)<=_ZgU2<&51VoGNY{fEpRaAVQ16Ek`l# zqSbG!xxvL(Q*Jp{1mEwg6N~(Jk>Vfr-(RQs{wLq=36q1m%k;Lh)H;Rs0StGP0nzJ5 z0Db6eiKtB3O1?=RCD`8{r}}$crsT)PznQYdKKW(Tx0~;iM=N=OPrlP9-=X9?EP3$W zf^Gm%_zgmgxaa*TPNw60N1E`9t{|eoU7?k)z#NjNSP18aD9^I&Ldh~7lZ^f%{V459 zJ5<@k2RsH}lX}~&Wl3J){_bg+fEwM+TM3g*U*T{z$%qpDB?`e^_5gU@L}`E_wsc== z0Rb>J-eyPkqO=s5!@j3zJQaA=@(7g%)p7;rlTa-?{!R6j9o^llmPt2JKoY$YLT_W_Q z%hs=6ZXLr(1wNuW*AW756NsxjF?>WV0s|0B-R-kLR{9_goPX7W6(Hi>oYN_Ow}$3e z+~-;G7Oe37#9@fMQZXNKnlA4g;(CWTO_#R_aZ8NGS&M;`kA*?NtX8i#6rSXG)^Lok zT=l^c=AbYS#Qh=Vqfl$kH*sS~F}NsWz6rLQw|4xV>w>!QZ!`)sBA_Cdz^Ws>9UYWS z!#;Svqp$L@OSGd(#Gh$olX1#>a1E9252w+EOxeR%c}wdeA^rj=6%DcloUZ2N7h~05 zjgAD%rqNOw;_eaq^*p>zwR!Vs_EyHj$KG!BU(>1jSGotN+QU6B@|!X8T1d3qElHTR z*`+|FI~DHsCN;_z<*kG6cf{1BPxr&P)HM;TZh9X*d=vm_i{G46txCJgT%5Qv^8k2_Lz>9SDY>+YwKOTAv; z7WJ}TkGahN_~W+%0C>nBH#@_BNC+Xu5c8IzvR@I1uA$bv96WQW3JQQom(L@(Kt%LO zw$yxHGVU1c3m_GiqmwUEZRFk=;OM(+B?1yO637!A4Re(sxF{ugU*%h25Z zU7L&V=Ty_|O1;RvI03Jo8amxO4sq^wm zt!7C`HNYIz+|IWZC=Ef${|~ZVzH&?!!-sBR7;c|tUEbkqkKsxrnFj@aqPD0daGAb; z2(3f4(Wc?swhEHi5wNo$dVmyl`a;laV=ua#I^0Tn6a6Av8+-Df_hoFacV7b&i#fk+rM&q-+UXLYN$BT(f~t5`Y($~Awuc0($80_ z{>#A`eS_qVg8^xfWh3CNA)`k{g#h$&yO8r&jL|>;*--J~7akRzsExQzNE{B{2rcU0 zFNH~PBIDbArnN%ajfd&ES#ZowiLE$v)AA^3CUaN{yL2=KQRLCS#Kz}6>{_ehydBnf zmbATkP5lvED!{E9cAZAL>?ep??y{^ePc3)Aa|>;mCoMnL|V~Q z>4@ZMOIRA`KkWrOHA zE&Z)yF8fEmLJd4kxO zVVe(_wBEfWNfMb-^bqakZ8%C!Ahsrw4_*+6rh&(4Q%;?BWlli06}Wd!2>IW|Yo=>8 zM(EJWDCM7!-Mfj>()TDU@s=FQfclDl)5<1aur~Q6?^O)8xE7WCOS(2K1C0 zq_)V(;u`?2#w|oqdyL#=s7$lm7FC)VJQ{=0l^bg3S>_VUJPa3phH1Pi8OLf?1=;kS z*Pr=FG}Zo?DMqbMr~qea0up+=Tb~qR%6eGY74GfhX;CF^ql7w2eRRoOp3KzMnk&9X*#A~OH!)eR^l*XOx5ufkcYEhocv`8LZS19_ zTTT=rAMpq)UxA{`gqU_iX>i-w3}U=ekP{UyWo~d*wA|%<$<)0l#PoOP=nIFFMtRcI*~^T&%HirP_^&%C)C0M_X6k?MkrMGz`hrZaB#&*jyW+9iC>- zYmFG{4RTNNC7R6(G5y_x`eKRiV??q*+RPd@;n3l3vh1C1D!(1GeW#`zs{%IjsO@H* zdhZXazHDdf)M|GqHK{MrPZ&G;z$mioTzm)Tb=>1CI>c9mCE|Mcif;B5z3{Q9xY~V- zXboke$LXOjRus2C-#v}n4ZC5iFLn#X)PqHN#l8-+7G#xC$S#-mbuNhd0H6CK{f!`7 z+)~GWzL~#_4BedX#`!+xA(93 zV}D)Bg+K9a=446Lk z7Hwsg9_hnfYfPhsLCWXO~qfEQ2Hyatsa7K$#i^D_&(*#*`yQ-#Bl;Z=>sP-m`pM+SJN zg_K10*zq4ac!9(iGu%pGuXc6}iF&$5YPDBks zA~7mECdECwKK{5Z{_>D2OR;y1sq!;(w%y|KR! zrYkTsIkLD=C0WO6N)ooR#fiSZpFv*AG&Pbr(Ko#WyM;8&z(n7sK|bR#C1Z%HDXHWC z=8{JGUyV{1&$%n++0ekF}cP&`IDk`Y|p&J|0G#ixZYghYrbN*DO!&^rwZg3OD-OL;HZ8!dvwH@hO9isIP-SM~D zU35n-`7a@PS6dE2qsz28mJFU_7jz8w^7!q;RosA>{)LM7sq{cx5`efB9K~gJ)JMnO z_J4o-|9-*QsgA&EgWPqrCP*>{6!SkZ9Ehz*X0)rxq|T9`$vsuGk^2+4AE7$+v_{Kw zTv_>z-NwM35}$(l#x4VAf(A}Kr}fLIOUL3Kg+)NwTCRCP9QR8MfQ@u2_6a*0t84k> zG%D6GsSvr8p|_jNm1)V0wgP3iQ?)h~)pCi8rdGUXtVIsbCOf!85DiR_C$7gL2P(`B z6gq2onT94o=7OE|+<(KQE5|#BaM?p+Y|BPRlC{-!fcK8v{T%C+=#^h$o_j;JLw6># zWs8sSbpM4dKtT7k?sBY>m_)&rSm3WX?$}%DoywYnb&1Fo!&>O(C`gxgFiCou8j(?K zw7P>dSA3YSUiT6FkMy~Y@Jfu3q$04=<}M`Jnr*kfJJDb0tq(EV6J=yNfuO1bjs zQ%IU?tPb=!PVbcuT;uPJ@VBvC@%}6@pEr2C>r3N@BU>bC8v;N26RT{8mE~)Z;UNK< zHKo457_%Yi9s@TTX`-xdP(b}y1sYXja>EYMppj>fvTFE^(D18MPGzicMdkr#q9rBB zrq;)+w2B*cn4}`6W41rnoITuU&XSFU&A!4bXsGZ5Zxg{naEuRrhFoC}e{nK>H1k-d z{XcKwV#Xfq@VEp8WXE{_g354D$&tu~SOGZPBX~iO@(hN&BSOAQe-$$GFRJ>I?LX_|?H3WhqEphsQBJ zl)HE)0O`=;HM(-K0TpLJ?kOOD3XqR#uF%1(1YgB(;| z&j6h#qd{k%u@v;sy@~%_GW%t%C<2|b0G&KL9y|_22t1Z&E=0K-czc0NH>GGj@DBKI z!GlJ3O7Vx0p8C8c*=tQziz&uN@lzbt-U?~o7DY4ymQR~ z40i$>Hh4p-Akkb5QaO>(kIz1Q+UvJl@*$5>4}Q5OlMT3yRIB@SK`-<`u-Bw32C)|z zFZbW*cBL+eF5GJ++2K}PcEF30al77P;Az!?ElI>sO@YhUYAKbtrtnqdF6K)^?0k#y z<1niu5-n#4k}KxVGBARfp_jA#eZ}qvn~L;x+zfEJ-2h+yZGA5{cDT1~ehYvxqtpV) zICnR+A?qOH7MP-v`BG}Ndb}@r-3RjUAIMmb?HBhj@91n$f6q`qCkWT8zh=Gq@e20~ z4a^Y&uC@k{7{L$YPRy&{9En_pOAl_Z@;&$j6I?p5Uv@yIE5_NNe2b1}d!m8axrFg- zZPj1wO4QEY<^^h`-9V;E?|L%z4`k}^F1tBPrv4&Re`R*>v2bATO0|ZAgGoclFp{2B z(pOaM_shA56;9Ei5WY*{gRIH8`QH8D0jbA z+DzqYR;eJZ6D@3SrSDl`eJDUFSmZ{NO^u9Hb{fghNUSll^T~fI$9|S$wL2@DV_wIf zV<;{PAJGAM)3mX4-k_G2`3TS1+Jp!lay|0vq7%Mef8Q+3ARBDC-&l8B7 zwJd*9>|-HIMuC4DGkN+eYo*?k(c2IG^Yk%%<5!H%=(1w;8PgT57#+ciQR5Jek%nRG zuNdtrVBUr;SPQj9qan4aJ;N%{4zd4}ETvUeeyPrVt-p@cFb?Cof0)#8{>v?=KKG1x z4U6iZZa!sMCt9qgrVLX(^^fTL>Y*Z(jObg*At@8qiMADLH7~Vwq81iZZJnrNirWDc zdyIueqH$VnrLJke@ChS>;gNdR-^^W;IcHYoF*TVpW-tl(3U1!^9`>ph4{RIQ zwx~+c1!GfHtvmq5Mn!P9sA`&i-EEwp^7V?rud_qBRc&!g*cL5G(SIkMGRU_Z_Lbex zavW(5XSqMi>uAFFH~#$qiLR)^i>}vcCYh<4Vgk4}eK}%F)s9lNyKN%j^&7-+!xUa< z^A#r3vy~`eT!wacV_4(qNO-wo&lRpz?C!LQZNO=WP<39l&&Wqim0iq$D$|Uz0@ms= zxKzso8ZkvXi~4vZp)HxJY731g7g@4UluV7B;$D+qJTqHG+!B2^O>DymFa$`m*61}p zZ>4~b-^$OqK0aPP#N*>%coYn1&B@loM;`$1f{$YbV-6pMlyxI$zY9JR!pDxZnZ<{y z&cnx}MxXD&M^&RCqg~*oN6v~7k*b;nvounn$HcNzgo%lb?s0uIO`Ih(crXnOu{G3i zwwv5Zj`h*7>|l?EKLO)+p8#(SnBoSLFCG--_=UYc;F4Zut(jIOP4R`d)IpuiPrFzclLa4^(G~P%3 zvRh(O6Pet>h#KGa(M1cbT?#TLhg5Kfm%mf9=D@-+R_m#>PvBB(e>X}uzT2%~psqf} zi}dGT$;h?W80jGv+x>&+n)@>vrY3FC)Yqc>%BiBl>idL%l@3ti81qo_nA#!(6iMk} zeVnZBcZbN*Xq+w42?p-MbZnfh@k2oCsExD5L@>5t%;+g#`DPpKsTId=1k<2d zCf=%R113w|DXb?%ZCFdW@#+p!T#9{NUN5=>6}|%VHH7z2_*C{5`7rGmk}`uAo=%6K zh6!^F-MkMyjef;{S@;*4C@YBeF$21yV4;clf@phM^mfves@7R>PC+z~a;*!iuI=V= z#bO7YU0BuH%_nc8YFpu%mtPV^%QmNd+4m#^N z&6nFn<#O8D4IzIrQ#%FA%?r_J;5rew@JwIsPodl@kdF{AbGKGJ4}~IFNHO z+Gx1~l*HQO=%0Ia_+(z(5#o6=Kp=sH4niECUS-oP129IB1+p$*|L&Xk2_D|+APMyL`S zfx#ddB>p6cH6_V=fAAAM28F+$!SP9=w|0n7_bj&?oW%5GYc&$l2-Boq z20t(1+@W!-go>YuteUH=86Z_QPTBB=`ZYpV9R3)7Vctl^My$YF zicRxR#hpOA`S?-wdHDGX^!>N^QLjzxwor-L@Uq`M2J*0q;b1r^0Ge1Ve5TdJV9bX} zO)AS=8GUj88KWKX9f&J)H{drr`@QXK=KJQ>2aE8=2by7;7(BM(Hx(XbYovI^r%uq`fSDIe#c=PLEg(1d&rw#%Gv1dAQ(at zYs+y31;`$A5?s;~JJs?GEk+1kCsS{srJxth|4h!2tpI1!2LPQSAeSx|8y;2^Pc)RZ zvdjEoXH|M&;j|^1GVBhV1Rh?hz|#cQ%x&aJPqTGR)-!+jtL3~wIlpiIa3h)AxAex# zjKgyC2cGO(1qsiu0MJGO+8;o(c6$d6X6^PjED-GWr3B1W*HVAu>|nraw_@XLp#-r1 zVOg*Q?y%TVKK2FzvfH&)KtA`0jhOAt>5)@pwu8{B)?w*djQ6T(>Dpks->j^9Q6SSY zhDv0=GiHwE$ToKSTX}Z-PPYD9n0c1`sTK)virA00+b^382Fz}^$VHYWA>!U*?M_C( zEB3q6Ez??Fx{IwoK#1PJRu6txv(ty4pmsBZ)Nbm!_3iXSP=(2Kh_*};ZhM)wj_vdW zHMqVT&DevCv7L_V76n~V;P#MGQ#I#eSh*kWYo?c`klr46t<;akU0^NKgQhWzNj=LJ zM90W*jprM|?ng#}jJ58YP+W&6LO;9M`*qoPBhJl|NDya5OqnnP5P^cZ|5ZyMK&h)^ z(K1R|N4Rf6feUkX{m}{!$Jhh3Vc4a?E*8W1@y8zV$FTTg_xNL|4D~12 z^&oh<=fN|Rxad#~^3dcRiuhSx#8gG>rifs>Hz{H}B64$EjYl>0zpMwm^CUcpw$ge) z9os&?dp)54_pS#NLuJZud_CZ__0|LCiS4nq?{g1Q3|sqY*}xO`R2B8?s#)!>*h7et zd40XuCud&wGc?3Y0hi$A_e%j8w-pTRksWO7(Il#_>yTqM?v=U``?}5617_vc17tDh z%}=x*fE_)JSlQENKl3|7VRyr;`F8YGguG7dVn_c9pc~F~cSq{VdUo_$a4V6kW4ui5 zTW!}-&ciLu;`Zx^O~i&)v!GQW?4m>|zr$oJpUe76o_%}!gAYtHm3c9mtykpzkm@qr}mQZEm;3U>htjH71 zFPfoaDD)e>(u3c@0&t8Tgdy}MgJf5V@-yWg;dNNd0?)3;_2>@kQv z!}rCe<(ySn^Z3ihWqXb_>YDp_MlMr!6T1)G zxB`=3Hg;wId}!Jlc8T@wD1a`AIC*XMgZcU@`{Qn|e+Sp_=L=+={tdbi=nc^9E1;b< z)=V+`Sg_kj`PQ-(kdp5~=G{&2LEa|zAba<*^0}i2i03rD%`;VFbYVnkJYckzLdR*c zv}L%Sw^)X&co;Z-P1EK;*1LL*vqd0ckFkIVLEuo!!o=)gi?yC|`iFq{ljny(*e2Z|0LRti%P| z!P}$nUnVlGz>h2gB+sC5M^RFG@fY^H(haeiy4a&~@qdRs8vOh-k5Idr5!8*lUmZl4 z$=#pi)BSUIiJA~?nIzTFm*lb>_>C{81~+M=8FwXP^r{(KvA_5*YiO-o5IFV%OC1If zft2u1j?bchdDD~h$F!q+Y)bwo+7cL7fm=5iN0b{x42;@nFQpd#=_z%I72ZtYEz5v3 zRds{JPt5Bc$$CL+={#!WMWj2Wmonxe;Ovu9Hia5Wxkf4JK2t=TblI8u$qZg*^|PKG zy6|J*P@yYBbtkA0oX(%39}dw&bJ5sndOHOCf}zI2%~V*BXc>x4l+5(H^kw5h-t{TYgG=vsb-JZ3RXPVtvIAE>;b3!dY*vEgT-?5@_Pwz$`Q`>5mu^@@= z^#0>yy71#UR&JM2?gEt)A+iG1gn+w>DzJ#Bm_=;%@a?fZT$XPS-=4LHe-!ND;Ac){ z)Uf10$2@nC#DdskAf0i03%n+q=@js8J=`SG=Rmu%Oo_w0kPD7+osx;DPh`e)?AXRN zqR2$f%r12AR4U1fn>+^Xtyabv5_(nc59PQ6MwB~;Zzy+08JPbg)u$G@Tlt9%$Bmi0 zUiG_6X-LC*y!~=MUkzhk-e&U7tDfVC!`uxesujwbyYn_^WBWLU7nptd?N#DWacDDz zt`@I8*5>*C`u?<;W8rlnx>3dk z8k)Ej?E>!m>>R(n;o|Qx|2}1Vf{5)_>EpZ2zeIC6TE^^?C74d`#y7s(JU!&Ug3sGw zcmF3l>{!7RSnL-bzVEfeE@=BN?66wP^xxZI4Ua;JtR2?5lefe60VBbb`c6A+4QfNe z=)@n;m3%v_+&XTu@&C>aJLeIZUOU4#SX)zOEAV#$Sv%}?v@0nWTDA$K{5N)(^0_DB z{a7#t=TlBTeuo{l^3gmy>@Au$JM3jXvUb>076^8j-UTpE;!KMTcG#T+G;;pJ@AueY zJa)oBvIT+fu)}(2Av9gs@Gu!SW`~_VUS!&s9rgrkjS|>%?RTYnh~M@6g0F}lHtrX6 zXT7TI>rEhM?&t-|Sc7}x0_Bs}x(Jjo9KzhG7X@7@kVI#h4)ye8(Ys3h`!0L$vEq%{ zd+(qobN1df)PSidu}?)Jg_}aLWZJ&57L_9HdJn~-Gx*Qu>A`Qf9APf5Ul2yhmj>N1 zfZC}2S59$npXAwpw3f5~?v?$g_H*{%Ty@dge~Sg=ciMjsdTn>-k15{O{(H!$T%#21 zKZ3IV^n?9p^<)2qE@bV$P~8bClvAD$hv=cX=$!qxgV}#ni2b*R{A<{Md*<1He9QjZ z3t}D(OZ*r1U!73l?LQV!qSIphyImE0Ja`y#_YJcKQKEkNhN;2!p9F-xm`lJ3sf%NcW@n0|{sZk@?7x2GGW!o4=i7hpiz31PW6?O~(VTpX^mL9J zXPxO)xqF`d$G7aiJ)qnlsb2OUKe3_opX@&xGW*YdvH#St>_3yU!TuvI&;C>Bf3yEo zB4__8(};3r@pe2Mr`_km_6(AiF21_p>G3l-mc@2(T0HE`NFqaYS*DE15>Rs4%5vGopB?E)Bb3fJlEhAsW; zR6?C_yuqO=?S4!&UvDPt)c=pYH-V0-$o_z<`*mf7&{^1&86GMc5JG2xgn*bp0tp1@ zA*i^_bUH~xT0_!ycUVT5i6RUrOH@=))aan&ZdBYxM&iDX#(l#vDr!(1C78LO=PU z4ph>KQ%L6|5u|h*a^-M850IoI5=e*VXW(t{M8XQ%ijzO8dI58~5KkxZa!EY0SMV0% zy(g3k{rxBLU|L+2!(v+Gmj*(u-t0YQjIBtpDngfrZz-aXL& zS5>yV!%xgsT!7RewWUfI|8I9D;x&(brKt{QGnD!>}Se7qPbtnh>uqV&syi zj6WH+hQAx722B|sgj|EpqvS9YuQ!cE1WV5WUpj@FREMYE{@r!W5ZOD!Wr^9Mcr9HJ zZNH~}(ZIe>{Z0XC*Yi25*43aZ6Jx^d5x%|D_k1tTvD}S%% z!|&Db>ycwn6Lk&cal58sQ*j@?0@L81Y{vgY#lM00$5UUe|4j1V-k(3xp@RSC)GvD` zfPZIiT+(&(^S$!Z?$$8*d$q4pWD716u?o=rey^4jSrhrZ-ruX?c?hb@M|C}q2!qW1 z;#8@9i-_o;bJ`D{k$`@Mb?-w2j24#|M!)YNBkq${chzlZBK>$4O(7wRy0$D1TU}A_ zufKY*-?nd!47$&Q2oT~ELix}N8HabhMc1L;>u2ct4hSjl)Bh%HJnr<#D0yavR6z&$ z?b{%_1pDw%1>2{G%j5bOw(s`dYc*fXXG|BNl;n$ja(PGII9{+rd)54|p0Ujg(_^Te zc`G$~;1%_vot_)lu>-aqc7kUGIv84Aa`iB+0Q>bAtybb6nNH^$HCH-_o zf03dWQ%~1UN&kC9wOuzr5yI!5f@XGIjyv>=bo!TK#5baI4SPh2R9>f0?_g@_swZkO zE5g@qg($SY{w~Jv`p@uF&91p(CsmpP*}*U5X&z*dl!XX~g+)5j38N^Y~t z9+&b*j#pD5dSI6V{EO(~XmZ84SVI@X)x`?Bz;E8goipen1iK)zKhVV&xR8BB&(y4b zG~p4b>CkaV1BWzlNCSs7a7Y7(G;l}*hcs|V1BWzlNCUrA1J&+;x5gC+`n?TH3JYCx zT{VpjxN52i>J3fx^?E~NP+#V*_tq-L)AZ^lZ+$JOb#Fthx5g9D7dQI#>UvK@t*2J6 zbqC!6PcWcoWo3yHI*z6QkvDl8f|K$DHBsqH-98`iim)$TZcZ#Z=gMGRV}riT;}3WN zmwE$B-NBl=LVamtZBxBRU+xa*HH}Ms-g-zwlBoit1LGihP!HC5Ab8N@_xU~e4m0uw zfTn`(pcjoP)JF$I1ImfE!CkL={C;rrqDoPbtFW-f?{Nn`F3*afr-6iV$vc^2sNq6O zxL4^w#RV zYBJ7T#4}Tr(`)M7{;{GnP-a3?z&`;^a@S9&_f}8v2{R%^ObB>`p7B0+%^B_`o&au0 z%};sUU&PKG%fWs1odOs zF?!=-ovIe+AXFo?QM6WdFY2cnt3Bj7hyzA5hSplC_OBfh=$11W&_+K#-)K;I3WfZm99pPJmmf z$*K!3tv^zB_xM0#lOMireBEVq6Xn~FbcTlwa!&$bLXy!Eih^XJqN+4XDvg9RkIW zG^t&4Z^>Pq5VN3_~dT+xSAb?UD zb+^6*4zD3RI`y!J9H%!@o0r4JRpXa?!LHh)&o5b^FY`7^p5%G;h3ID*%%nvagRmlU z?wF~TEP>w8-3_P;x1`}@F&GjQQW+$tJcF7H^CS*vTb7AOMIR`SN;Zd~OireW=0P+g z(obP-X{rx;$;n1e5z-@p5Ngtw!xZ%f_fn631UrK1necl{_2YEqh6%XHxdo$V}n~X^Z+D5T}pg*A@EJWBKr=PsAv=TPy zs$96BeBP``wL_v_`jyjH20g{pU{`UVQiL3RXsLn-|KSb<4dKJ08?J*HjZlt+O`Tg> zSy?=*R6k)JUUOPrQ9Qk(R4+N9R9`q>FP=AlVOi+{y>!uxQs=_*`SYeRmh=0IZ+lXb;<8cRwl4i+Y{G2qiTjwlQd4w~6% zUMVaT{=>D{gJ{NuN#E7b=wC{6B_@BrCxB3a+^uqjZa6j!Pq+{mNR~c-LHVrmdBqia z>4F9G7oaxAX1JS{)CG0;kEVt*8XA{3=nY^3)A6jR@%YGSN14z>+ysx`tmz32Ij<+w zz8YK4fOYt|ah{CNrRVm9itu^ioSsk>=%U-l!ZI;^L8-H%ct&Z7KK*18O&mr`q*GT@ zfKUPK5OJj&4Hljq(If?8Dp;UyZ>)yG$d`HKh$EZ8c_SQymm)QH4Fz1i4Hc7(j7IzL z2`g9>k1nMO{O5YZY-)e1=0bgLdF9;Vg)_=%T=&X~KKxG%XzWM$ufFyog9)?yOFTiB z4090PlTU%4^sznjxw;V|?w z5yYxvlu|&MJ1`T=GtB*&)am2Uk*sWUqnCWg~nJg`rR~h9;C7Mc=#LE9^sw zPz19g0w%HOXb2kJ@>Au*GdKx>5WER1b^qe}#^qG_C+al@vm8dF!YwtXDtA4#V5JKl znnkaB3?~I+5^92HSlo!^gs%x9o(F3i8db_B#z6_Sg@tN8Y-qRFu~7es@?)o|MHA%y zpRMNvcZ9_7ePVzqQ!){w2@#+Rkrr0kn45YDf^-8)X}}s#1kFKOQDJ4}S?UYIX)v@_ zl$Am0QkO62508UhHcPzqF4Y@;k667F7WT(wQoaDCe|`JG8*N>9bVN%;b?(I;cy07s zBZ58=iHP81>Oy2G$ZLx$YU-&X=v{~~x3>{Szc=_+y4*hR@BcS)x&nQKg4q2-edZ0o zNQ~*CR!vh3D=3;()S!znG`${cQbgHU5|Mjc;gSnlWBkr*TIyU`s0ZrYSg49Qa>zb9vFT zF+0`J?JwzBnI!Gg$F+@3SlUH&dbqDiextuj8f(gH(OdBD0q+td%ECPz*-C@G(eshS z(gY|EOC4@3KpSgB-Ucyf!~tUwR{`V(#L^Ji$A%@C<)V&L6$Vespv zkY*Ll(4fO`^)78%8p%AL$zpXFbb-1?B&xzxh$E{J?$&#Q z@Bnngpc5(A20b%-0+pauURW=bXN`S;uD7Xj#37r7g_(zP=tk>6CHVD{iU~zW-pc5c zsQckNG9S^{KnXC}MMAh}eo^$oV2uK_>JVo#&-MtvoDz23`p*;Z#Lzjc7dZkPjV$e>? zgG}?nwy6}yz+9w}l#f?qQzZY<0$+e!*t`=eDoE#sAJeD8VbNhezWXM>|7XKo=B1;m z#8iQ{k&^(B{@oa+5vcGzIr>Eupl0Cz$l3nq!v% zJ1Fo)^^1`;UBdLlq=5sHqxL)kO~`cku}d}q49{$G*HbE2a91Zf7V~wDa9O?S63V|d zL55}7E|xhKtg*v5cI+9;>1<3An~%*8p$%+9HZ>q6g-xTRB$lNMw}V{}ByZ)$DB{n? z`r7e<;7VkeMUs(rO2#*MmWv${x&1;J;%X1F>a`y1Tp~T@sm;|fAEClcrC zC|8!Caw;-y=W}BhR;5*R3OmGt8x!J~sC=i|yNN}gGX}Xx5vL<-EwqN&5L*w#yP>gR zyrDpqi$IQ@9U-SUcUZY%R|&entRVA(QgF&%8O5<>8*xQB8EaSa3NVihl1#_feLlTL7EScC6L(5Z(Jigd$kUDe+gEXxRHDVixY@X_j zl=NrG9aXI%eyNe})ja{^;xM6#O#zj`icFHKJ)K6s$SIf%WZEH+?m#v`sK!Kt%RTNh zFvqKH1C^l|ot5Ko1hR~Ai{)cDlYyBYVhJN8E&ILV=>mgN)Dk&Q$M74%QP3y}?_3yz zV7%Y6g!Zv$&qyrHiKTI~H_d>J!zNEaEO_fM99z zLXq(=^n0NZ+ANef_!M~G2xS?Zs0(KnFD$Mo#2%uufbiyE%}TkysFICDBA-vFK@P+3 zqUR~#&5$7#8UL_9F?j0~=fR!zJ`Yfcor>k*R*20g?2j6HjP6V^L11gj#12JXCMtay z)<1t<`i2r(fmEOa*o z8<9RhsyaM8rsz}|;*hg?tH4xr^eVL?|Mhtu%58~^Nn{cVJ4#4kXCV`nIY#87dTal` zrkCiTSw0a%3KDndQKd#dGu$@PK!#~eReKN9GHLQ3!weT5&__{DqrM)kkIK!_+>OX$ z45RVUXS0`98CalZLR%OjGd2JM9#Pbnc&iiasglu$+8P!)a@MEWO_aZCGD;b9wgIAYE$ z?49n#0|C8CAcujyAL)0fd~Old+2q@^n;Y7#pZ(&w>BvvawHXsr&o+7 zs0Zbg$B)?87qtd^ds4beddjd#Je%N#2E{{u)Dfe^gCM=JB_07uN~D#%Z)FyqRrGt* zdG9GAc-1Nd@RAT4nUpv1tvpdS7iodh;WI>}AkwwWRlHz9@yV`YXSwR=nb;*gVL>VE zkT#pCae|fJO9-zo;vw0DlgqT$zzfb4@f{i^zeoo z1*M}B46;=D6G3yEf}Ry(_oBRF-ei|+NkbED*=FasD)3|ywq0nJD$OY^ zHLSQ7&YV|Zh(1x&T)JS+yh>NTlA^r6!kay*v{YcTUFBYN>0OE_t=e14{>$=}`ADjG zz17twBJ~P~vk3lSnG7Ko%&E+FRpz?9-g*+n^e`(Q*(OJCn=126IL%zPpjSg-2?B;z z=2e106u(+SNA+mkFDp-t>Dv(J%w^7A0Vc`dH)o>ohGq|oME5c5W0I>9JNUHqhYd*z zfPaaR^wyo7uQo&RRI7S~1foXV&lVd7E+JaD5fS$t6p2LLjwqkw2nR*jf9g*uyElSg zzEncQHe|TgUQ=*J7{GsJ;JvI&&;DEfKX(8`@(-KeFBkx(<)aMNU}rV}V1K`E0GMj6 z7x@eM_ZuLv@5zz&U5l+#EK$mT*`R?G;9OeiE?-vq%RK13%3!`rINR*(91O1T!}R4f z{A`nB>TJ6gVd1IXgI$?A+|U z?1|Zvvh%YGvM1*_aawg_X%E`|u$eEn$$j#2p$<58p%bl1zDK|g2Aa`<} zBQHBICoeZIFK=Srq`dsRg1pHS9TT%B=1k0;m^X3a#7PtLCl*YcJjpRBds5D%+(~(p zCQh0(DSuMIq{;b?{OtUk{M`J!{E7LK^7Hcx@+TKK3bG4w3UUkb3MLjzD#$M=D40AM zB2Gs2lfim2icJP8N)6%5|9`fx`*^&q0mo+{3tCDad35i_+l7wD`5XfSqCXqw1nue~ zAqv{mHND}94&&OOH?DPYmocOKu&8Ux;o6j1&e+YsJO6$#)Qt1y#}9^X0Biwl2W)%s zVCX1Rv6-Gd=mG8T9}MYtM!2jC7sme3R01*ijd02Tr62dn~Q zXje1f2*6fA2Vgs331BDSB0!ed6RHQ)0hG&HDD)T72XE7hztg=G29Ah z$9oB12Xp}L2Xx}SdIivL3tlDI4A_RZ2HgT!g;!HY;2;vX+ z5(eY`G~faC;U3C0y*$EQ7^ z?SS?nc)c&iV=G>0(*ful5engDYpiNyD6|8xRS$*s0=69)3gwN)^=Q1A8L$&@2cVuA z3Vj7wbS%mp4gD8{LN@@m0PZ0;ITXq`2IT=AfSpr9As?W9S}4>8*nCDPWXr_;dXxuj zU5-Cy1?>1^DD*O*?`+gN2K0Xlg|+~;t-(u^06W%(LYZSB=S87VJz(?2kO#2+O56wR zybAY^1>Fs}Pw-CE2S^9ZQ5=>(E?|}wcFW-d;uGlq%g{_lUGwBl_(B?+DfXEu!)6ak zUY^j*jz6q$Z0=Fem@Hoc*wKD4gdr3@i0^6mJPEu`<<&tnL40&ePbfYDw+`ncfP)!} zgXp&7lL?$IFlaYm8$Nl!afKrp>9G9KSs_{wf5j!IGj_Pq4l;jgi(T=yJ%bJ^OR>*O zNsx0G+M0oTa9`q}Ix>JGdILvop?5(I0FHQ3`IAtd@I|DnC_M341pHLs;ReJ(a?}G? z2AocqiWK`=>wIvL?OKO>6;brtfm;-Vo?a(*8t%a@iG%8-cTd+v;W~i}M&Z5!t~m;q zU}bDW6mA4?TO)7|)ZqYb2XG6i)|n~x4c3yBVQaNnDSBg4%CO=T`}CBA(&RGZl)-En zy=$)nWv&XB!Tq&bNs7MOb_Pn7h*EQt3vj;$v>BMEzaiS`DfW%lSt-LdXvHb|T3c~S z=4xJ&;;>$pl#&Tdamp}Ipu&=5Q4hTn@pY7ajLJfkwIthWZBB}Aoh5}UPDz-R%&0F8 zfcES+4u;%B3oaY1);~~ZkUcHLy>`fpp~Y-h!x0ExOGICh4)QeEAGPV6zLfU6A8i^e4%& z**ZOC*d}dyioVe{JtcDkFHUi+jVn$mSRG%IQl+i8q!a)HErJ4+QgLq)of1a^TI3iMp zQfg0G-}cN(aau#B3`>(4+0pB0&j7@CFX5cpvypVOK?AR_9Ti#KQFb&ZnH7P2A4^SXU|k%>yyCf$cy7?j!fk@_Y2y^f@Zgw{ zY^{iD=gef52Yx?--y+0*ACdhSV_Ddwb~L!C(L6Kx?C^PUvJTQ9%Gr}J*W=vwH((5V z7JdM_1AZ%L>J6HU`_mMG=Il81J7}&`^(*@o{aqUFZ!+B@Ax^a6mL}6!YzL3?upStP zbLwa1k1!U07nZs-8Oo5Okxd0z9?Gx*J)z4`uPt6z{fp@>($CCF@ri2tIGW;QDj!7o zD#W^+%D;nv1MTl;A7v>;)&}Sul7zbgzQ<1GG4VO@ay^7t;-InoBzR5*?zb2Rwr!H< zCbHR$WV0J=C1f)ggRA3arLw)4(So4ZocvJ|`LNEGIpoRclEm^2b%A`*ooG5wy4A+*TrK!+qhjo)C6j#d3V! z2YT{1mlORlpcj1yP0_sIw3dcDuQ+*O_#C5)d^fGpc0GD9gwT@NzWXg=6X#lOgH`mc z9X@;u_~w($AK+f^*pD2$Rn~8##suUV0e()DKg-~~lXzFH=4)*mw2hL#5BDEMJ2w*l z?YM8&_hhoU(c`dK%x`TdpOMiM8b{@QC|^qR6xvPW&S^bIwiWYKS#o{&9I}y)_oHkA z)=+LLTO#Y;KxJv%IjvWO^#n_xw(LOwQ-(Fx4>*@&)OtaZm;ssK_ap9S;?>uy)P3bo zFek0GSr>sDbcfk>I@uM~KQ4)}6`x?3lC;2n+it}4>ZrdWut1%($covO`avBp>mmFzfHwukmjN#J>nYVz)a-2(D zi&eiY;7NWn2%bkgb1(#xV74SFSLEC) ze3zIn#oRkTrNb)pcMHlDym>HmCU}T)eQcg)cj=_4BC_*6pig>O+4+Z%OxZb&@dLPb z3uHW-bo(Oi2{}a^0y|%gvFWtlfjC4q482&MY%Ph5OHc{ms%wJAMKDpuMO# z1vX7OX@hLG-@v!woceRMb(x6ENbYBGFBA8Et=zL4Plgz&#bOb6tN@>Jm}h2U-nN}?v~$M5I48ApBi|5* zMsBfEBgMQ-?YsqL5;2~KP~9c{+ezb*@>ChfQ+;7x0LL6w>Jt8Uc1pWd zYs7SH?o}kuGSs_22eD`Zc-Hpk*&C1c^5k)*JjMJ+^Tc-WESrKH6ZyUiB+m^Lv#+J~ z>}nfAOhkyiE*kDa#BVR7TorPKFHpHNWVy&#P>gdi77=LBGN`|aiZux3G7liv_yUc^ zQzf5%aVlKJ>HTJ3>-kYAv92MT@PX%Dh_|1^x$Ob)?484AA((78<}mX8+d;c%dQYf< z>c3LfzuAg7TiL`+it)?B>&iI@Y^@)eI?|HlBJkS}ek*47gw7`$d&|%Z;?a$?=2da+ ztdtImb}R*7nB&9o03vU>hN8Ufqvgnx!k^e)He@ohQ_;_}Qkt#S8K&JlCz<-V2nn9^ zPV5Oik8|bg)Ve+*=TMql8ICnDwrG920_8@WguE-2iyAAv@xw|qBXW#Xh&pLcp&4cG zq_V=likJoi#X1-D%n*UhtYiu(z!^ae1`cLkwjbpZ7WITyt8%NY*7=BQ!v3iO_gCOP zqH$*X2D%aRt0b9;S#})oKF|a~vyW`^RjI#l4BBk5ekL1=fTK+K$yU(kAwT^r@q7mK zb7_sH;(Uw&>nSLJdQd>bS5(K#DAR#5E2s?F7+Mi&x3rj$p=oh4(c3bx<~$WSX*$2! zkO_Gskz?x3Fp>Wt`aICLfqoOwZ}@5c)W@};x1ZJ%`X|vZGx&=(sBv7>7#UhoJ!PP! zJ)av;?lJQBzmw%6=TtbrwUncwIknj$askn6Ix+4@hA!|L=jsXFO?sc(M+VqA#d}rO zd!vM)ays@Ac2@O-DoKVD`?MA1*1`u>S+A1iq)L&Hp>m5*ZV&bm&Z7E$-={v+=2naK z%t-H5E^M;}<&LVw7zGd8#Qx>rZmbARXiXdrc8>z>lPGsK%FUv3DJUmw4~vk58OiAH z1PmKihkMGY2N)95nx_)cg&xHkM#SZFQrfIqj`3oB5wBHBo{?jycPbA7z_G z?OkOqoA6V_^K7uW0q=}Do_LP`c|2Qwf+wbZ)&kStaz6I8U^{dy#Ah#9w_JdAKL~6C ze?~p`=@luhV>Ms0scysaPeolfTmj!a0DKdF2H(yhnj`V2d6N7`-Ov*{iFkg8&ijw! zbBOkQLTsM(X#Wm`%69DG_h5WTeu$wqQC@7Lt#_^zc3P6+v!0Q%VmAC~S;{(mS}f%$ zzEk>P;8;weL0BE>dm-v;zpE$oJJR>cpV0=T@2a2BH?^Z439B;fPYgl)@-y18AZ5iV zXh(5xJAT&>6SE!TP{jz;x9E|c&;s&99PJSP0iIXpn%2vyTI8BUzNP~7>mKb1(SN4c zK7n(Ins-IMhBi31M@5I2Gb&Ku1}gs;_D#r-Jt50)ru`e$SG_((#8z;hT0Xgy{;qkt zSd;DnzdG#c{+9S{ko?TYK3M&+e;9167mh;kQ}k3%D4+P8ZphUe^YzO?Ai-)k7mPx5 zF%PW(&vDQ6gr*=UwoNg3ir7}zWxsi-+os);X4=?!$=XbFuYlUf&+J8A4(#*Xi=2<` z`2ORxJe(_6dTq04+u}_vT#S7kJPVMvy7=ycp{GeFKY%CFa9DH4ZQvVio3zbV>QCB> zU-t!`F`;-n83Q}!SPqZ9b4*)^cF^2?3-~?xbx-Jes`Hut?TFGDO|RQb`4vNB=KUk) zw_4OS4C&DU2YNyp$$r64)fJZgEz`PYCR=|ry&?Akno-Zr@70)@BkNIfH(~^pBmapp z(@y1oz#QR_|`Gk>L~_fL?`NhVvdBX|$?^n~80F??iye*JWez%z!ei**4w zu%mSF#hcO3QU6a9T4=YPq0mPIz~_Sgaz*(fY^qqbb+n%E z?~G<9YtPtBRSs)&pstcp&_ik{}yq^d|ier|M@Q~Pqw!K zS-^9o^g(@N@FMG>uK?^wYB#_KL6&nw@NYElZ!Bh2mm7mVw9q zB^XyE!-G}8|?YDXD(oVF?y`i6L<82Rje zI&?(wm}yl!#P|n~VSfpQ;;A31z{70J^{)TsBN416a#Pz7^Emmt)}xURzw2P=Se(m# zJw^CCx(wl|<%rLSD=qwfjWt{O!y0?h${D*#BgF06Y(YC91*qBF!>SNZprdi-lip#dA=Tv6!>c>Ai~o;NyOWb(n>JY+-j<_`4Ri$5JSY z44PcV*k3GsgN5B_QL^YYjIFouO`_$tArhNh`U*_v8Y{oqN>v$X`;5!1c>3AOKe4iP znj{_YfrYQLvej0;NzkT!ZQ*xY*+o{4KU8hA@`nWW=-lOu-D}~WTiJ6K4qd!z;pbS{ zXC%ds7Jj}}>X(!=eI6bcg!EE5PX3LJE#+_6*d`0#ZDTiE_;oh+v;|Qm>$31KG`7yl zU(?tX(45BZ)n0|%;F2&8dkFU1Jwp`Mnyu(aN8*v7N928~cZCtl-a2dISCn3{D=(@3XS;C_ab(z*#l_g0r*u zKF+pScqeDqTlj07*uQLJcUfU3UC?Pf>#_1{G~N==9@h9@;@LkneqJ1V z-UgPxM!K<=S@@4u>SO4t-NG-nvDYkoyN!JcRoh^Vyhmf-TK^(-mE=Mi9M<;t0aT}B zD0Lh)=y#-J4?oX_2K}H>gP?8j+^6AzGX9~J-DAB=a+p>G^7lk--&*)ZaqKTvzCMm! zW#!Fr>~1UninC{|yo<9Btb7k=-B#YgS*s0&p5gdHtx#V$*hT$cOzcY{dKbM5SL z%d{8ltVLT1%yt`p($21mqq}#<^EIjL>-gW`{JI4Gjh!_o68`4I5|mn(q~rX%qzXKH z`g$_IAeG%U;0+5)5RO}JEtUBXlKtEq^WsgDwR<=ux#q5O` zf9yUhze!`y6Yec5zd~c5(&=|r-lnk&Xdqpx@f{j<0=Xd;x}wcuaWCZ8S%zN#)3LDU z$v!t)N3~no161-y>!?jO_M$fGfX3F79-pv{dW5qp`KWsm@f6mmPvY2Xaf4bD*^c;8 ztqJV=_)$+LvNsY^E=y$JCLF#yk=>DqLIYUxs-#tks}feluZmm6SJ_r+tE{UmtJo?b z1@=J|pl!CnmXhvxVKFhVFVx&0fnpaqJB%e>#Es z{GkLqB+2(Au=N@TzE$HNBw)nyuj1HljlYz@It2bBjW=V2X#D#)w${ei#*?Lh{yrOj zDUNm8ractLZsq(wl;Zs9IJTd2U=DJk*%HU^OJFz0@vb;_R~)}Rfqfp2hC;T<`G7xI z`Gp|T_z!WcOXK({!%a5AT+iwIY0kfnW1nz-UOd|n$1jg#&&A2Y_gj59S!dzT4P%#D zhJH4Tt+$SNa~K)*Wy9Fx8h>jj+pqCW!`Qhtj=bq5whJ`eejuLzYY1DDz&{$o?oXHu z%<9DLmdA&(+miU*8SJAZ4%9bEw_7$3W;+M)YX-9~2N3Gqfj3%S7|tF~;UA{6YX))D zch4Yx_Yn51IDczU8Hje;sqUxj{H`?ijyT_IKkW8&wj(ta>~^K5fy4W${HDR|lT^NS zF#9$&6*X^7ZD~|ZC=HF;lFlDaXE&vD6b+^GTQb-s8C30a8GP+vwl{+Wy?h9N za0q@}L4|G}%3mDH9vVt*x@#E!5F!jC`~}1L7sJ^#!}-49?4IHL?cwa%;nX6io<90f zsH=egRb%g1`CA%Iaqnt0o!+9ce+a7&T`|fs>@Xf=(&o6c8Rt|mje#gg=hg<%NM z0Y>&gK<4s!tel_30yqb5cxpLgjrmePsWqV zve+OeApHM57C2}*gO`*8g^A`h8%tn&cRJ#L9PT>@^~UM9nsG9Z%cXW{p2&V>>kdn8yCD9mm-98xm?oXrw?q{Icn{q@K1u586h*l*n#}FH59J@zDf!O&tGc z0(&ryUy;Oar>Y)|=Xika-&D!dXyN9-+_>Uv_Zk=898=P($$)6g@c8ugG^yo;8rUN5^ z-J|0Rx+*%~@)TnD9|n!Sa~QkT&aobX=em11yFZn84P)=7@^^-@kJ4H}ec3Sn0BDAB zP`)_~h_>O1$41KsNid5+qyI6KJwJ#)HH>{Sh_?@8uiN>TLy<4x9}i{Ury@dT*AC_I zb@vVBp!{Gc5DyR|$bq(fhPK_CG5X<=Y+nY)^PFo2^KVA7uLtu_N3tu2@cVRjooL(s z5&U+Yol9-&`7OVHB-=8QqtH%jTNkzMS=F{QM2tv4A^^rnK*)S2M%)#{Evb`Vv$R+( zJFNUhE6xALuOT5t>INFq2Q4>P5L@j~W`2Z^zmJg0a@fB#_NIluqG3#OobR+w*bm>K zhK-(sk{y&xy;>Xby2hT-_{SQ1Nh5E1CvigE`)!s^+Xl-ImUm~biQd8O&oh?vWqebf!Q_W87oNdIgDR6kG*;r-!zXs@EiWY zJof2t_&syk2fsZ7cV8OGADGA9(fNT{?Cw!6V6Hxbe?6bQbOZ_7b;L0c`pF~T*1+%S zV@T?CnG5%wxO9P64~?j^*FYW#1l4cOM(aAD+v)#vKKk@5c{&bRK&sYuHtD z*}t>+O>@~f6Zqxj?5YX;ujTAR2M6nK9Q;xzYsu#4JK0Uy{8}e_IGaB>pWT|nAD+P; z&*AsaU|-}C<&AlV-7%YOn@F`hJdv95;iT~p^wE66Tsrwk)VOytoi`Wq&Qi9kkecyk zA^-bq)-BFMg`;ry?kW6>Quh25zOR(MHiiFVCi{2_mD(|tH_u@APUT-sXKzpCUDMgO zQ~93h?84*t)*0+?$MFp_*h9zhj_K^R@ zm*eQ}7jb-d2EQbp-<8gL;`!Qi{#XKU%i!37MUz^A~)AX1-=D#M7(jQo4dicU@LdD>MH&3M6n_3-zCVTk zkiqv*d~je;8j7~s)4;yno(2&*?P(CRIW-N1T2qIjK~JY50nwbsA4+3eC~~?cjbERJ z0Vg76+@SHLH9MYd(6ogd4NE@12*SY6uKzqAajq`tt~&L(GQqfj2`J9C9+N*s-# z`$HP#VhoS5j2evxiq6!ow^`1$ayZ5ht=i=_%ilFx3a_q6vkxZ`+kjo zimVE8en;b9s&x!XXO(@A(pU;@iev9NsEnt$thre$40IWjr_NYwoaY! zCb&Ew$6p!F?ug%J0k|}g|7$q=65%$$d&&G$MB@W~1EN7&Ep(pBA0N*Cnzr5o(4N7c zM~T54+v1N8;cpGY%<&SGyIV~>`9hLxKT5Rvzbz_kpqX3Vv3bC=Tf5eB#qZ*ujbCFuaAN#x$>&12f2VMqUp;8dttZF- zIEY_$a(rtleZP{og67Ij`bmVdmE?HbE>E@Ur`<(E{l{bTvsIqZ^glXsM} zx5n{3tnBs9(0f-FJ}|~uIwX#zb1!woWTB-bL=+@ z2Y!;v@qHkduQ_4hrFnc!`M}%q`0f)1ew;TCG>=T&YuR%Gdoh3P!)0tk!Qfrx>@Sn~ z{d3vXlX+V?dvP+qb?(6K$^51h2d*pRSDiTU!9ouD%~Qsr@V+U0x07w2%AcIiuA9oA zn#bOqN}_C6WGR1pLHx_5)VR)4{>@48`%C$^Gvn9KB$_W~^3}8A zH<$4@XS0vX_><0o&E+KVWwZHhv)T69sATtS{=#hb>}=#{5771PbNGdG<1ebf>E5~g zh52mXTx!v6^Z1*XZRT-k;Qsmi4JUhWKFPJi$v<`u{JWFX_wWL~Z$bR)3&<4isN}C# z#y?R>a_u;Qcbzcs?PRnwA_r$^6;@Y|8*D^uz%E@F4c~0IIf!oIKkykO)5*$gfIaKMdrT zq_B@u_@+Uuc@T;Dw?X{%LF^xcW@4w>&i`&_H`+<2Z|(f`!EALZe||8#6pmpqYfI(- z7|fnd)s zR?S};$m;ny1F0*%9*Caecyq+<*2Dgi!alKDx2Ld|wIMiPXVWfDf%@tEZ(Qpc$S#bl z$9ZRb9=@+k(7sN=>l^6&V1o8k3SQAL4dIDSd7ZvaGkZXmw8=?iVJPuT|hdBc+N7c1px1@1P>BAhJd9fR4Q`9~S- zO3SpHGT8eT4T`?dItu5{S+zHZuv<04f2nC#4`$cc2>&mecKr~xlN0`Xu61Xy8{-K7 zVVrjR5cXL70~{BB8%W=m4|3xB^FjQs4E6%@AQ^05Dt}`zTc1Yf+tPT?VD^wWe=d!G zjXqB=2L9{xjQ5AImop&WhQavirY~wn@n$sR6-&liE4y4ZW25B^++4};NN1N>if>4# z7Cf7d79551*R0y=bWC!DUt`lANyk!>@ZZ|B@6*_O{CbWH#47l1M!E-}ErtIbRD%fh z&>((C8VQf{&31l68WuT|alSQm=+$ZL6_{i?7EME6NoRQ9 zRZ?DPsl-h;zbBpjdD{9k_LxPx0<0~gaDJOryEm1crxE^XO}j0XRwG1X)7n$nHGG>5 z7g$H*`<~W3r$4RPfLaGN-*>)?(`9G}M6RAiKj>7rdn%0rZ?ze5T;o{o_y4{gT1ujj} zZcby5CojPH`hk2WFew_&uS((9rm=Gd5#=__i|7M!{`eq-n(S)(WZ?JMGoA+?!#j-~ zAgSJ0Qth%V#ML$n$3w;sV+SCOR(1EKBlF5%Poo`*E$IxiJc>Q49rh@C&Su@4#`f8U z;QUIiy^)Sz)6)5_IPJ$Y_P6-kt+==~`6PUA9H6a39}mdG`KJT4Thj5vmT5SDYoPU6 zNHK6B&hNK#d_N)jd!IOmYx-w8+m=fBzoqi-GDms{#Hnw$#5A`{ z=*Yv8HsV-!xTMqnA(2i69ST+{SkxX-$n5kcWSJw5qNwZEsO!$CYn^i35l2v9eAhGgF#ZI=%ew?G4O3M_;*I(Umq^@Xw*}5YhSs#qVBgw-9Hd@zccE7 zLR35S+xqe!7InWl>V9U_{r0H)hTPG5G4Qcqo#NY}?4(;kgWeZ)-K^;C4@wG$f(Cu7 zqBpE`_9%JBxIQ`Rx+d!S%&6{3brfQ zpg>Q_)#(4kD=1!K3lZ#ROF47} z9SRmHn0d6ECoC3=Bvju7%aT?xIL)5bmzMN?6$tPC5)m&iNcD1u_#5co?a;xUECL)HS{c9}%D z6@ItE+uJ2#jKaUK@cLGXP`B9v1dxKi!e0wII`A~5z_(m3uf_X?ar%+M>kmo%JQaWr zM*K>A^s6NPB?(xy!Z)vzh;a(PMd8`i5}~SL4=DUCivDm#j~@#O{;i6h-V;g(?d(&% zPejq5ukf9%f{Yo{?VSq$UKIU13V%@H>790TOhJH7{O{Nxujw6dbX=#!W4$hZF9(u3#i#5TW%r8Da^TJE=Ul}{SN5aG z8Gd9T>UBopFH`vF{J^cill<+*cvFn-7x*E}Smzl0I~5-z?_l8f3;My#ST`B?Sn?+) zP`zoa`(33NqX3{;dxwLb^l9WR=zT|Y6bnB1wb31t&z~ily|3`b`nXu(GqK>Yu)|rA zYA?vdu|VL#N3Dk^D*Ocs--#a(&|&a@FrWAgWriObrs(srpc8U_DiJGH^R_B{$8;(9 zKVg4#;E{Ntf4f>R8uI8^2oj&xKS&1jdrvy{D}4Lw@|xaHOGi2uwnSf)Bk5Zeek$

      5KPgCxX#oML)Mm}ep zBxm`+i}rpc5f3Q*NvRU=qhD6yp!Z79v0mYeu9kRurwScA6~1ki#M66w=op-)_^A8| z$i(qy;Hh0LQSG`+(L2i}WBNTb9rV69lG8U{;*V1D;OP$V8NsTQUuqT9j6EdqXq?L5 z6)8N=kb1x)oC0I?`-1{Ml(ninqeIcZ3p~}^{H#P=C!5FO2NR#6tj@?wD*7UUhn=Xr zTd~5g0iO6&eI^-z=H4CsA@biAtYzDMLWy`!EEdR-6E8+lgc23QR$5_}d)Li(i>9k(gGk$0UY(X1E_ zhv-{7C4$~BOb5N|ity|Oc}?#rr-NR>MtJ-Al91k+M91{u5+ALfB?|9T%C&qxHlqlS)whfyvJov#kVPNo4*e2jeIQR==2c(P|B zE;P#d6}^LgS&f6Nxl3Lf{kTKnjXWjHSR4-nPyWZq3qxJv_?Mz@y-r@! zdt~Xz8zK339FW%wC1CXbI8?7w*^hGDY^%aMzLSLXPE44&#`qa|LkW2Y#5+h>BDWhsAhGw{@2U5(SN@;0l}rJh@( z{Kk2}6aVHnB!7x8>9|an^edw1?*^XeyQA!(OVHby^DZffq5tMls=e`&F};hGjwb|u z5Hs?#63f^>6@B;Hl91j*Lr3}%l26ANiC1ogX>6a;d%8`F0VzUtpn@UImAIl$96+8V{DM$tQ-lZ5o23v}#K_)dDU z3yw|V8&4b>^dX6$_wmwk)X}ouXusrA_^N%9PK812uL|EhMcRX|>U~q;nd(5bEM()4 zk^G&bB)t-g)hc{Dy+jR%Vdr-%eCPYJ-hoEDGNqn7qUNJFfG2%+eklp*9XWK^$EbFx zeuvt{F$s7YM`g;M4L|m@!W;Wr6BWHSR`Q7+r}V#2#Hams*{;(SeTTvu`%*AhaeSlr z82cy2I7-KYg8J8?`d3r*Gk_Q4@ivJN@3zC~I)!h3TjJ?I0O+`1@PXG<`C4PXenru@ z7D+;S7bYG16}~D%;_2PLbPOFQ>$UHe*M^_WQh5DENl)+pqT^JB@645q>HRfy9EpAq zC1iW^^7_*RGH-zwQLORnO7 zxkRX8#x4e)_#6A9m{!DbkHYJ!Ukv;kicc&3{s9NQmyZsAp6p*^U(lF;Hvv!WjrQkv zD!e^e@}YNUivt6V=(`6=e1pPIRCwRf5`U*6Zvvji%c3ZM*rMoL)kH+^U!`O8B-vg^ z)Vk(1;Hlou4oPR&VPd|dZ&l;ku!r9OPxQvVr!I@JLP3wXSM4t<^|0S7`p#)metKsu z9e-B%==I276`n=SQ}+t~hqJP%_P(p=JF+F?N@f2!1xkLk50CB>hgaYaXX~Q)tQYv9 ztSoB&dQRbu=M07^J~Jju{rjT$xPT}9Ty&u%+^FPvK;fg~w>JbIJN|1>62cvcBcV|8 z(NB@>qIba3QK|5qK8dGytJ1L(c;Sa{lYGvUXf|ew;uGbsP6VFljpr7M6#eCjz9njW z4V)_Zw7e$?XQ_U?K;fh3*=vC({>{pM(iQ*v6@7OU{~^aI`PFzoSJ6KYycicBNkp}( zcltDmca+KYj#T)S3g4_8JiVWdj@uO8rxFkUlxX$?@Z_Icl-;7c#o;@i@Q1UisQ%pm zJjv7ijHEkT(cdEIVgJg{8+P)PLI0{GRMQySr|@kbOT3Y9nO!9HP^I!h#=P96@I}f_ zrpsb%0{ke+Z{(Madb5fpJ|W7U{{TG6j~7n}4lc!KnV`pfr2NBK3O^1NiT1Wj#8nFK zRQT4ZQvdXBG&)uxz#w^y{I(&__!*Lq^Fc|MDgnDp;iL0E?Z6WsW|JX_he>f|M z>X!+?lbq3csy{1ybl&ND!DlGzQvIbXc|KM2%^ymHVQ;milD{u%9{EP$qvxHGGfB=7 z;Rh;=emntqlC$k=$zOF3yI>a4V^OG{mr&xcYXp8MGoG(8^w0skXm6C?8&#(GN9lQs z!gr{3))rO4%?hupfV5rVI~Bg_6^R%k&4o=bS9*9}BJipMarhP9uIe?$`(}l2R{0Gh zK6wv#G5;$6kf7>aJ6rOPo(CTQp6tPR4$07iWsan8jmoF#z!UwGQT^*t^et*#L+^Q~ z;}HX|CJ=g89vyfU0MN zHT><|x#skz#=xH$1Ai{?W^&^738wON#K6BBgU^I{=KN2If&UZmW_s8d1Ai6pLZ5r2 zezqy)?TcIDF(h7cr*Uj0Z)2vc}gnyBQ-8|$Dn@^_)+M|P8D|{+!aUC0(1U(z}po~B$yrp zUm1gsF9!bH7oy(24JuIhcd(p(Sk#K3<7 ze42^;ISbA4CjxJ#=hI`*x5S{o4)|1)dLNHL|4Iz}zhmGJ0&mu?V@|*;Hqe5qopQoa z=Ez*Yo6-A$H*4>CG4L(Go3-mM;74H|DTs>eK8eBSN8rumDLqm4yQ5PoZ>iGZg}|H1 zd1VazO~9M+e>w*KqZoMmN#=Ye0B3v? z{14{(xiSX+;TZV0fj84z7w{x!^t}H77T{+6^$PH2d_IUlUtDF*=gb)R+hX8tZu5GF z18?RJ3xPM2e{Kx?RWb0l15fkplPV8mwEaEcjdR(<^82e8cwTKTj~jT3ADY#EhY>$) z1K!L|o(0~l-Zx|L&##gFtG_4Zt5w1-0^UqNX9I7>e?9PK^4uH)|4@o010&mvd z6JpSx5rckn4E#qi_#B8qKg(+_|DqW9`WX1;82C$pH?za1W6-}E1OFlLW_q)pZZ7`_ z;LYe~#GqdggZ}my_5De?biVR^ZLrdq42Hk}VRv0=!wh z`+*mFcFA@b|C4xcqd9$&&%E9Vz?<=369c~kczdLP>inJ<_$OoF_r$<|4!oH>wlmG~ zN5#PZ33$7SoR`MH-w*?T5Adlbd|m=xH^ILbgU?qn@HRj7WB=m_;LYUE0p6_M3gFZF zvtip~@VO%f{z>4?+WT<~dTYSEUgkPwUS%-b;VN?n>Vod-dQZ7CJKGgp>GL#JpYEv% zRtEEYZm++5PGz>MGB{}&P>ud_=ez>n%4%;zZFcsgT2IjJt)EOqT;7Jojir_D^2!2N zX{k7w?kYtE1umB)Lrs;QdN9cj1WSSUoY~~AcLf{E!5-Inbr3_G&#NqPWf!>SHrBfQ z?uI2E7v1qTECII?SB}F~;#ur&st?MWwc@UE%jx$n^|<^VD&g`q)_ZGKid)(Fy{&Ww z{cdkCP+n2(D3zaV2waK^T^_$X;DIdJ`QC=0XNe#3)igE)g6@W3d38lK=oVDwy0W|h zzdOqz^_TB*)vQ>Not>Ri?GAWrT!Ek;qL$ZyNp?-0+i%oXLn^5$Ep=Cx&acdi0FVU!)Mt^PTvWVK}R0ex>KUuW7rm@!3ho&<4OC`+C zfhI(={Y^E&a_1aKy=-Af`wqW)e`jYfMsrTkrorxiXVW>6_V~*!W=2jatYpC)8XLlJeEvqC93WxiHhcxfUciHa<}Y>TyQH5% z*@(`Xv!J(3ox7p7-jiSJse$<~?qka7q|2 zwcZ*}U_q}!@`6$aKXGvhpW8axSAs_Yt^Ng_F_I@mBP+ZPw4-{#oQ19h5I>kb z(bZeR!fZBxAmQ`cfwou}UC@dsS+&dj;8ya8S&0;R;B zRWW~hafQnjXsULV8~3xQi0hQfU0`aq2T4*&4LRil9|B0!t%Ia}XO|`e|?rD}ojLXid z^0d?TI_>JN6^G%1R^r3~A#vq0TwpH1;88EF7(pgF3<7nYi!JS6ZCDTxK%)s^wg+>UVt2~3A^r>~x3B56`F7}+pO{g5w2%Bnsh zvF+3uR0q>5!YY8F(|FOD0C+Z3_rptitT2t-;2JL9IFEr90P+^o!6a4ld2|^Cp?l%Y zpOoXr6R>MDzCqOruHj&UkqA^x9X>vedf#S}wgScsfq(x&P(1Vj4eF!|ZhGUGwrNZ! zGdw*TSqy7~FM+W+^BT0y#}ma=4UiA50JwXt1*W03A%0(=P6Fb?I0Ft|m1da*fcKYR z*Rh(Noq<~^;PN<)x|dxRW*J~6sb5j8tlGm^Kw$SWdf*!{5U=_*NtwcEXn%nFTYg%YdIDBp?;m&pN|SSDA~Ob#W(iUB&a;j>gF zgZUiI0n4y!{>IwCGL!N6^c1yarK*63;647r&we)Rvr6ke(}975Jp}`Gst3o9RrGWx*s}-o)|G%13*r$5g^kCp|Uw90J_c2mL1p`2zfCci3I@lv^MEbt<5?V z|5U6)p}ImHDu7);{veR*Vicf)mp-4`ao0SZExy0lmhE}%xzZ10N37U+g>Q8sA8e}yZ1K;@s-&Q^a zv`+_MuMHIv^TweF)`!A>-kpxN3@=z}l#O?GZTAtk zqOJcVTkRX*xN}2A3=e0ES&{(=h$f6czq6%|U_iUiICmODq%9KG!I}%G&a*`<7!Ulrt$K6tG354vLOl{uU0~Y1 zrpgF|hJhff*kyEd+*ay*x?)>ssE#0qjrlbM60N{kHyWq&8LX?c#{0ePlrl5m*|=EF z>fv})o*z&`IW(fLyU$v-1`T1R>45ZiH6FbHp3b^qaYfKdQIZIg)(htajC@y4S?;1r_mTf&3DxfjgJ_w!2?CrX7*681-kNUNS4>6 zC6ENp8%%A1)YYnUP#;g(@TwWbB1)pB@QfTt4FZ@J;}G!}6J?SR|@ z)`Aq(*gO=~p{=jtbU{8Bc+u##@fkRay?4~bd%iKki7xtxR6ThTjo|AS8kHPx z+@Ha-I=7dd$V1jK8Dpqum;)W8F?5%;rLUoVMjE? zE(i7P6RrgrGhxhJVb#;6SUwF0usTLQFNq#c9KAiNs7o?0M-MmKjRnewYO^8jp%3DV zc=NtCqdD#=!G&#W{aiPV2HB6^f_6-zw;Lz1wBHCs9i1pchCj$6(gZEk%nKLfnFPWL==Dq}> z(3k`m7&3)~CC5+ExLC_(SRuHNQ24Vol&O?To(WR@BAO=Eoq0DO2#cc@003e;ikL0r=AkujX_g@tV$QntX zk71#^oWTwZ(N~ZkgzQep$@^K$#xE72DA1}m^(bI=)$D~;rac}b}CWcG0XC* z7>yeHx)8(eN-Ll=kS}TU=nW~&*)&-_#x*b%T5AnA@VseZ#mZ*jeWMI~H6nNeX1CN7 z)OocHma&=)u4XIpggTufU}OB#-lP|cTXMK>51_n?wUt^6>}^;|7$TY{8MwRw&XZy6 zFqfUE(s*(T30k}5NF9XJ7EZSqnlT;<+&6p%;-!nxr6I~}iJ9>W*wRj`1P;ICGOA5x zgAp*H1vxo3R<#Px8DiCT)FQnBFKo$pQ%kNx0^_`s^cF~fK&fwi|8a?CszK6?5P=sG zH}$N4E7LeFpuo?$5yVd4*d?|AQ{&OS-6&Tjl#^*@5K%0%f|My4q}U`lsb*jRdC-Q&I!z~}j$6H(gvgCy-)K$N z7Cq6|7^?xJ@+NUpsOH3Rn`lB_P84+L#~AvSf*_FnO>EdJY7YY}w#@JBXQ`;b)0n9} zK@EYf*rr7Ll}MXL^sRP)U^RMT9*uHFDYuwh3q35)w*&vLPMlcYvNo+2Wdu#_@ARxU zK1q=|#mCXEE@-TEUhIh&W}OImzr0>2ei)B8-31GtakRza+9e3js=I=+juuqlw?T|V zu1~Ew7-BC-ig*!*c;h&2q34RwKClg~f>5>0J+wlE_mG)0d#)jY&5Fu$bv&(kif z6e)t24jS?UI`2vrqtDlcC+>`O$VOnIbNjv5KFbAyA}CaSjR=2vOa8UvxO3c!9mgDK z68A)Atz9Q@Cu*Jyw+Q=+3SBR}2N3MzX6Fmz@yz*>u@vT1fflwd)-kFlR#_2x+$pRo zHxP^-Z!PfN06guPzracgqOtP!_9BlNhj`y=+L&<^O`DbkKG+y-GK%LBcGyG!Gpt8y zx=Iidi=ED7P&yTBm;qiHJJk=%#1o^MYb6>7R7;=$M<70V?QFysgW+hFVAE28TRX^k zEChCXo6pwl-Hww%lL>aP>rJyHiEguFE6qD2!rSlI+JO+fg>E`-{i=dgZfm{~15}~? zW@lNft^YXzlnWHl=8JH{9z4&Ac;kRN>)Jfr1+d335>xp0;B)3-Q&Xk)eD&Ck>xXH6dp_e-wF1!Fy3J(R`GID>X>6KY=*VWWt!r-!vv z{d6{)q@9OPB?6SM1p$YF4kphBS1FWP*g0(Yr+eR~aM1&=1ZRRFd)|H3c}ORyqIalt zI4PZU%n3b*J|@)kz*t)RP*a^f;L+h>FY2rQ-u)xl5?Sab*dTFS0u>OAuA3nXt_4S% z)ctGKke65I)|sD`n(Tapxnr~Qv4va~OA7x)CYl763rY%LDRn&UW@q>!lgika~e~{^V_v>6+gy3v@E^GM2w?;grJ^v&9N)jP#M5L(U=KyTBTE z>AAP)Y)uSYXN9$jp-UON%LF^(TI(c&(Fzj;o*(13%~Py^b&1j}kVY}9DG{ZJsq5Hx{wb2e+jT#)L3Ex+%gkiHx^(qVy z0dG&5lD!Riv5=ZY967ys*#S%7E?9-w4-U7%XGO}ejzGIHeAH8gyH}4mXPP>R=L^mf z1O9?!1s}!B3*<5D!^)1U(TkyaLYosJv(X#q;D-D(v`qxTAPA6*E}a=F2US#fG(%dS zy_60(yMMKe_b7Xi>LKq61^`plwpq0#JeDJX7csKSQ1ebH@s`cl#fi6g{{@@`Cq1=3 z^Z9seLu^dSj9cgz@*2fL%WE~*z}gz z(!4Q`+Ge{cZ2}89{Z~-d@Ung1DaRw&%)l1l0w01CNn1irrbolr400BAbb60>IZ)C+)3H z$&ZNLe!~|7b&8}`Q^av2%+jIx(gM4UM_zloSIF^5S+@2@BbirS0|GsH+p>2zI*c?K z-X&?y$AiT*dQnQ^WSGpo-%(_mzebYHkD#@v5u9I32Haw!kT#{P0Ee-Ji|^$5-$1z4 z&4kN86%#J1D)FOGJWUriI#Nt5$ z4T-WbAv{xt)1U;NUK{>y{q627TQcaRhK1<$mD|cgE%$2I1g{sDCU^p2Xr5;+^-7T( zIY#2No+C$uwIaY)_X~EdlfGOtP z$w=*)DhLivyzwbK9^K)maFWt}<8!1JrR50y#aoW(Apigvz-&SC2&rB)1u;C(UCj-X z-&q-rtqI(P2Krg(ekS)V9w4I+=pQ^|129-e1$QH+MMxizN_QN;ApWhvoB$cjK$aq8 zZRH5tIbTs;(!nX*2=0OudiNGkF>iBg(OYKRdh7yXu%*-><>aH}(e)rD_9-O7 zQpfIGmIJ~`0NEr~1ezg#3utlH_lU0CmfPxQlT~`LPBMTshrEdcL#AcpJ}A_%Q;L)K z40|N?CTP2pj>hdJwH{7%EtDp+bA*VnJXj+O<^^XEWLeyeTT%Cl2@T-%V79`Rex^9W zx--B|-qcSU{nthW)O?_~q=T!Oj8Y;yd1Q(FScPYS$zT^bGn7!U$56^)8)K4T;=L3+ zLPRo=;Um4VPzaHT&mX8Y3#ik(a>&wVer0DqsNq>1@nOk9L>sPZzOzprf1b-5AwMRJ zcC~R;QFoVd+vDk& z5)}gn%@R8)4E~inOdip0p!!jqAV)M-lB2w05#e!9prgV@h#4>l3<6;vxdMs52wtrZ z3AP|Da|<~)pCi%KG#cXmL(ZIuDvqGj%Jku?NF=oc?VQJ`L4+Y{8~a@TE+&Yz7I}hvPD1g)aTzPOI0D>@!`RgI<80negm-t&+|^LJl^shCD+;>c1j}B(Q6r zvhtyp>2nVYqZyhHLH~1ru<&6YWUkLCu^%L>5fZlm1ML4MH92w?0NXE6BolDRAdG|o zPz*MRHE$HUS0BfNXHd8yik_duXFi!YAbf&f7@n*pp>-QWxZFh$VEBVMYuT{RZmlvo zx+XGRimr+HQ5dekH8$i|M{+fbR5>5pCZM_*r_wLQFm(e;b=6D8s^b^X#~Z|RXo`oT zr)9Fkvu%j|38>UlNhl=Fn{%7nChAG* zlrgr&x^pFP$D>IEC}^6bgozogQucQy54XHegSSn#@29elk~K?JCL6%*4a4Ly3ml~4 zHiSBI##l_QTGe|8TFzcM0PC1zjwMM-$SU^)W;HN+t{`)Sf83zg8<}P8AQ^AQ3dxM5 zzIp|8yqiBH--X*~PJp~?niH664zvQ0&PA==09&(p6m=Jqkz*c+-@dC}p>h*vrQb%- z&L9!(KE}gAav4k|BK4>N^FU@G0_v3U5F6-D@#Y9%Q%Zfs(mKv8CDo9ttReG~1{y{k zXW_E}3{Wdx7A7(P5d$i~!e9+72!R9dVz?(9fY2czZv!JkjylcE_zcCqNI{` ztzJO6>lq**F*zj}XMnJ{1+?MgmK`jcvR8TCN{F>hCc5pY*qSm=L?`f>p#w<9SXc(^ zf}v%mAX0cv#Jp$p9kSP2t46VO=LB%m7(pN=PgRaj1uZAaqVR763NQE-Q{NpL5+$1eu)@y4#ljL5t4&VW#jL#Ao zByT%yT2tl(5>b-S^!Y6!ySybIN4MPeL6kryG8HY3kpti#my@7yC z(_(UPs06SFH{w3+y?cy}IOo%NB!;ED@FT8jli!fK3Uq}xE%}qqq@b6k)n4!OEQ`Sf zZ)AC)trZ%-0mpQ&j3U05MpM$ri3+K`G#RdCYIL0<_-M#zo?o@HM!I*1CU)^K1r4HW zWNJZDndywSXR!08U>0dnn)i0_t3F`BrJx~MPX^{O5yO(c?%vS+Xn!`x>o!Q?10KFF z@5~ag2AO4ET$JrXqNoscHpieZ&q$B4Pb06q5Yr*0bH!cj>9gE5pfyoBoE$=trVzFhL^vXO z6>{Anmp)S4U7qw5-jPPhbAW*OD&0g|#`#2E2to1hAn!>Uc@}9b5hFVw$BpR~m-ZIt zBnoz!Mu9;aq$)Elj#MzlV&-6GMqP2^&uFKQbi(%CIoJ|Efdi;7lIAgot3q(5cvZvu z5BB#F-=bgDc5T>7g9BCqjx?z^5Ng6T44=Ua0E-0ehOMOe417IS8wCTW`&+yqiPWV zQwyq%Fs!y}jb^)AL5*u+F3elD6<_3J5U~@TtOFc9;%o1@@JF49V3+68%U%%OR4koc zr;E;->r$e~(jGfe1V+gfF|nH~E`5ttQZr-31Q0mXLJc5owDKLqJeXO+u5}C3E}drB z3=f3hSu?;n9-}VDizQyj=#Rd|68iuLB}Ig1URmXn-7e$Qz@gw-AY`kNP8t)I8#VTwV@ihjwLKGAex=~2XSo7@ruyoBLT`H$n30`i5<8mR# z3-b+tA}%BAd70jKM^2AY*W)3N7OP~Qaf;kAI3ExmWN-CQ}Bw@tuFl< z?*3`~>DRHB{!M=|$9uWjekONb#<^R*^fi5hK7K2A|4-!3_i+9{Uw%0=&wo#sGspkX z_nUJ0=YNRbe-?kH{5$$abNu|5rGlTtf!+*k%InLe9IQi0PfA@FgS95$@S8UpEo@0&|xcz^f@}JOv;A4(I&<~{c)1PTS-9w+}_@lZ_ zeedkkRlF-#AK!ah&du{pdDH(t#Lx8GeYvgF`nG&A$I36rIon5+MS zFK+q&{*jb7$A6$j=X~7^(bE(w*x*YS*%b)){?j;hDyHPGzK0jZk zygB|C%Kp4l{u5pPlN+9I%72Nn)PM8*3tj#~m;Yxou(*dlrktFWN^hYs(Vi(^`G$O} zyrr+`@@5)MdDE{yMOpXxm2c|uZ_Duu`qBBA^5*!9_}wjEx~HG}jvU{!j6o_f<;?M$ zx$^J5tIPe896xiOOmE3hx{ 1 and self.ddp_config.overlap_grad_reduce ): - # Assign a communication stream if we use partial DP DistOpt and we - # need to overlap communication + # Assign a communication stream if we have multiple DistOpt instances and we + # need to overlap communication. stream_context = torch.cuda.stream(self.communication_stream) # The RS/AR communication stream needs to wait for the default stream # to complete its gradient computation before launching the next - # gradient reduction collective + # gradient reduction collective. self.communication_stream.wait_stream(torch.cuda.default_stream()) else: stream_context = nullcontext() @@ -314,24 +313,21 @@ class _ParamAndGradBucketGroup: local_data_view, bucket.grad_data, op=reduce_op, - group=self.intra_distributed_optimizer_instance_group, + group=communication_group, async_op=async_op, ) else: torch.distributed.all_reduce( - bucket.grad_data, - op=reduce_op, - group=self.data_parallel_group, - async_op=async_op, + bucket.grad_data, op=reduce_op, group=communication_group, async_op=async_op ) - # When enabling partial DP domain DistOpt, we need to All-Reduce across all partial domains + # With multiple DistOpt instances, we need to all-reduce across instances. if ( self.ddp_config.use_distributed_optimizer and self.ddp_config.num_distributed_optimizer_instances > 1 ): - # Create a new coalescing facility for the inter partial DP-AllReduce here + # Create a new coalescing manager for the inter-instance all-reduce. with stream_context, _coalescing_manager( self.inter_distributed_optimizer_instance_group, async_ops=async_op ) as cm: @@ -366,13 +362,13 @@ class _ParamAndGradBucketGroup: communication call to complete. When ddp_config.overlap_grad_reduce is set to False, makes synchronous call. """ - # If overlap_grad_reduce is False, start (and finish) synchronous communication call here. self.param_gather_dispatched = False + # If overlap_grad_reduce is False, start (and finish) synchronous communication call here. if not self.ddp_config.overlap_grad_reduce: self.start_grad_sync() return - # When using partial DP DistOpt, we don't need to sync as we launch comms on a separate - # communication stream + # When using multiple DistOpt instances, we don't need to sync here as we launch + # communications on a separate communication stream. if self.ddp_config.num_distributed_optimizer_instances > 1: torch.cuda.default_stream().wait_stream(self.communication_stream) return diff --git a/megatron/core/distributed/torch_fully_sharded_data_parallel.py b/megatron/core/distributed/torch_fully_sharded_data_parallel.py old mode 100755 new mode 100644 diff --git a/megatron/core/enums.py b/megatron/core/enums.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/__init__.py b/megatron/core/export/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/data_type.py b/megatron/core/export/data_type.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/export_config.py b/megatron/core/export/export_config.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/model_type.py b/megatron/core/export/model_type.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/__init__.py b/megatron/core/export/trtllm/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/engine_builder/__init__.py b/megatron/core/export/trtllm/engine_builder/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/engine_builder/trtllm_engine_builder.py b/megatron/core/export/trtllm/engine_builder/trtllm_engine_builder.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/model_to_trllm_mapping/__init__.py b/megatron/core/export/trtllm/model_to_trllm_mapping/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/model_to_trllm_mapping/default_conversion_dict.py b/megatron/core/export/trtllm/model_to_trllm_mapping/default_conversion_dict.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trt_model_config.py b/megatron/core/export/trtllm/trt_model_config.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trt_model_type.py b/megatron/core/export/trtllm/trt_model_type.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trtllm_helper.py b/megatron/core/export/trtllm/trtllm_helper.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trtllm_layers.py b/megatron/core/export/trtllm/trtllm_layers.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trtllm_weights_converter/__init__.py b/megatron/core/export/trtllm/trtllm_weights_converter/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trtllm_weights_converter/distributed_trtllm_model_weights_converter.py b/megatron/core/export/trtllm/trtllm_weights_converter/distributed_trtllm_model_weights_converter.py old mode 100755 new mode 100644 diff --git a/megatron/core/export/trtllm/trtllm_weights_converter/single_device_trtllm_model_weights_converter.py b/megatron/core/export/trtllm/trtllm_weights_converter/single_device_trtllm_model_weights_converter.py old mode 100755 new mode 100644 diff --git a/megatron/core/extensions/__init__.py b/megatron/core/extensions/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py old mode 100755 new mode 100644 index 62336cd..a89e272 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -13,8 +13,8 @@ from packaging.version import Version as PkgVersion from torch import Tensor from torch.nn.parameter import Parameter -from megatron.core import ModelParallelConfig from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding +from megatron.core.model_parallel_config import ModelParallelConfig from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.parallel_state import ( get_context_parallel_global_ranks, @@ -654,6 +654,23 @@ class TEDotProductAttention(te.pytorch.DotProductAttention): else: kv_channels = self.config.kv_channels + self.kept_packed_seq_params = set( + field.name for field in dataclasses.fields(PackedSeqParams) + ) + if get_te_version() < PkgVersion("1.3.0"): + # TE 1.3.0 introduces precomputing max_seqlen to remove unnecessary kernels and D2H + # copies (#555) + # These two arguments did not exist prior to 1.3.0 + self.kept_packed_seq_params.discard("max_seqlen_q") + self.kept_packed_seq_params.discard("max_seqlen_kv") + + if get_te_version() < PkgVersion("1.10.0"): + # TE 1.8.0 introduces cu_seqlens_padded which is the cu_seqlens with paddings counted + # in each individual sequence in THD format dataset + # These two arguments did not exist prior to 1.8.0. Full support added in 1.10.0 (#1012) + self.kept_packed_seq_params.discard("cu_seqlens_q_padded") + self.kept_packed_seq_params.discard("cu_seqlens_kv_padded") + super().__init__( num_attention_heads=self.config.num_attention_heads, kv_channels=kv_channels, @@ -683,7 +700,9 @@ class TEDotProductAttention(te.pytorch.DotProductAttention): ): """Forward.""" packed_seq_kwargs = ( - dataclasses.asdict(packed_seq_params) if packed_seq_params is not None else {} + {key: getattr(packed_seq_params, key) for key in self.kept_packed_seq_params} + if packed_seq_params is not None + else {} ) # overwrite self.qkv_format depending on self.config.apply_rope_fusion, which can be set # after init @@ -692,24 +711,10 @@ class TEDotProductAttention(te.pytorch.DotProductAttention): qkv_format = packed_seq_kwargs.get('qkv_format', self.qkv_format) - if get_te_version() < PkgVersion("1.3.0"): - # TE 1.3.0 introduces precomputing max_seqlen to remove unnecessary kernels and D2H - # copies (#555) - # These two arguments did not exist prior to 1.3.0 - packed_seq_kwargs.pop("max_seqlen_q", None) - packed_seq_kwargs.pop("max_seqlen_kv", None) - - if get_te_version() < PkgVersion("1.10.0"): - # TE 1.8.0 introduces cu_seqlens_padded which is the cu_seqlens with paddings counted - # in each individual sequence in THD format dataset - # These two arguments did not exist prior to 1.8.0.Full support added in 1.10.0 (#1012) - packed_seq_kwargs.pop("cu_seqlens_q_padded", None) - packed_seq_kwargs.pop("cu_seqlens_kv_padded", None) - # WAR for peak memory usage. # See https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/merge_requests/2388 if self.config.apply_rope_fusion and qkv_format == 'bshd': - query, key, value = [x.contiguous().transpose(0, 1) for x in (query, key, value)] + query, key, value = [x.transpose(0, 1).contiguous() for x in (query, key, value)] # In PyTorch, the following two tensors are in fact the same: # Tensor with shape (1, S, H, D) and stride (S*H*D, H*D, D, 1) # Tensor with shape (1, S, H, D) and stride (H*D, H*D, D, 1) @@ -760,7 +765,7 @@ class TEDotProductAttention(te.pytorch.DotProductAttention): if is_te_min_version("1.9.0.dev0"): - class TEGroupedLinear(te.pytorch.GroupedLinear): + class TEGroupedLinear(te.pytorch.BatchLinear if int(os.getenv("GROUPED_GEMM_BatchLinear", '0')) else te.pytorch.GroupedLinear): """ Wrapper for the Transformer-Engine's `GroupedLinear` layer. @@ -1229,8 +1234,14 @@ try: from transformer_engine.pytorch.attention import FusedRoPEFunc - def fused_apply_rotary_pos_emb(t: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + def fused_apply_rotary_pos_emb( + t: torch.Tensor, freqs: torch.Tensor, transpose_output_memory: bool = False + ) -> torch.Tensor: """Apply rotary positional embedding to input tensor T in `sbhd` format.""" + if transpose_output_memory: + warnings.warn( + "transpose_output_memory is not supported by TE's fused RoPE and will be ignored." + ) return FusedRoPEFunc.apply(t, freqs, "sbhd") def fused_apply_rotary_pos_emb_thd( diff --git a/megatron/core/fusions/__init__.py b/megatron/core/fusions/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_bias_dropout.py b/megatron/core/fusions/fused_bias_dropout.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_bias_geglu.py b/megatron/core/fusions/fused_bias_geglu.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_bias_gelu.py b/megatron/core/fusions/fused_bias_gelu.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_bias_swiglu.py b/megatron/core/fusions/fused_bias_swiglu.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_cross_entropy.py b/megatron/core/fusions/fused_cross_entropy.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_layer_norm.py b/megatron/core/fusions/fused_layer_norm.py old mode 100755 new mode 100644 diff --git a/megatron/core/fusions/fused_softmax.py b/megatron/core/fusions/fused_softmax.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/__init__.py b/megatron/core/inference/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/ammo_support/__init__.py b/megatron/core/inference/ammo_support/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/ammo_support/gpt/model_specs.py b/megatron/core/inference/ammo_support/gpt/model_specs.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/ammo_support/gpt/state_dict_hooks.py b/megatron/core/inference/ammo_support/gpt/state_dict_hooks.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/common_inference_params.py b/megatron/core/inference/common_inference_params.py old mode 100755 new mode 100644 index 2235308..7955bb6 --- a/megatron/core/inference/common_inference_params.py +++ b/megatron/core/inference/common_inference_params.py @@ -1,29 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -from dataclasses import dataclass - - -@dataclass -class CommonInferenceParams: - """Inference parameters sent along with the prompts - - For an explanation of these parameters refer to this blog https://ivibudh.medium.com/a-guide-to-controlling-llm-model-output-exploring-top-k-top-p-and-temperature-parameters-ed6a31313910 - """ - - temperature: float = 1.0 - top_k: int = 0 - top_p: float = 0.0 - return_log_probs: bool = False - num_tokens_to_generate: int = 30 - - def add_attributes(self, attribute_value_pair: dict): - """Utility to add more attributes to inference params - - Use this method to pass in a custom dictonary to add more inference parameter attributes to the instance you created. Use as follows - c = CommonInferenceParams - c.add_attributes({'min_length':4, 'eod_id':153}) - - Args: - attribute_value_pair (dict): A dictionary containing attributes as the key names and their values as the values. - """ - for key, value in attribute_value_pair.items(): - setattr(self, key, value) +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +from megatron.core.inference.sampling_params import ( # noqa: F401 # pylint: disable=unused-import + SamplingParams as CommonInferenceParams, +) diff --git a/megatron/core/inference/communication_utils.py b/megatron/core/inference/communication_utils.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/engines/__init__.py b/megatron/core/inference/engines/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/engines/abstract_engine.py b/megatron/core/inference/engines/abstract_engine.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/engines/mcore_engine.py b/megatron/core/inference/engines/mcore_engine.py old mode 100755 new mode 100644 index fe81602..28ef46b --- a/megatron/core/inference/engines/mcore_engine.py +++ b/megatron/core/inference/engines/mcore_engine.py @@ -3,12 +3,12 @@ from typing import Dict, List import torch -from megatron.core.inference.common_inference_params import CommonInferenceParams from megatron.core.inference.engines.abstract_engine import AbstractEngine from megatron.core.inference.inference_request import InferenceRequest +from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.scheduler import Scheduler -from megatron.core.inference.text_generation_controllers.simple_text_generation_controller import ( - SimpleTextGenerationController, +from megatron.core.inference.text_generation_controllers.text_generation_controller import ( + TextGenerationController, ) @@ -19,7 +19,7 @@ class MCoreEngine(AbstractEngine): Supports any model that is callable (Accepts the inputs and outputs the tensor) Args: - text_generation_controller (SimpleTextGenerationController): A text generation + text_generation_controller (TextGenerationController): A text generation controller that will be used to define how to preprocess prompts, generate outputs and detokenizer the output tokens. max_batch_size : The maxinum number of requests to process at once @@ -29,7 +29,7 @@ class MCoreEngine(AbstractEngine): def __init__( self, - text_generation_controller: SimpleTextGenerationController, + text_generation_controller: TextGenerationController, max_batch_size, random_seed: int = None, ): @@ -42,7 +42,8 @@ class MCoreEngine(AbstractEngine): prompts: List[str], add_BOS: bool = False, encoder_prompts: List[str] = None, - common_inference_params: CommonInferenceParams = None, + common_inference_params: SamplingParams = None, + sampling_params: SamplingParams = None, ) -> dict: """The megatron core inference backend generate function @@ -54,13 +55,19 @@ class MCoreEngine(AbstractEngine): prompts (List[str]): All the prompts as a list of strings add_BOS (bool): Whether to add BOS token to beginning of prompts encoder_prompts (List[dict]): All the encoder prompts as a list of strings - common_inference_params (CommonInferenceParams): The inference parameters + common_inference_params: Deprecated. Only used for backward compatibility with + MCore <= 0.9.0. Use `sampling_params` going forward. + sampling_params (SamplingParams): The request-level sampling parameters Returns: List[InferenceRequest]: The output is list of inference requests containing the generated tokens, texts and log probs if required """ # TODO :M core- get rng state tracker + + if common_inference_params: + sampling_params = common_inference_params + if self.random_seed: torch.random.manual_seed(self.random_seed) @@ -73,7 +80,7 @@ class MCoreEngine(AbstractEngine): prompt=prompt, prompt_tokens=prompt_tokens, encoder_prompt=encoder_prompt, - inference_parameters=common_inference_params, + inference_parameters=sampling_params, ) self.run_engine() diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py old mode 100755 new mode 100644 index 4825dfd..ea0d67b --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -5,7 +5,7 @@ from typing import List import torch -from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.sampling_params import SamplingParams # class syntax @@ -28,7 +28,7 @@ class InferenceRequest: request_id: str prompt: str - inference_parameters: CommonInferenceParams + inference_parameters: SamplingParams prompt_tokens: List[int] arrival_time: float status: Status diff --git a/megatron/core/inference/model_inference_wrappers/__init__.py b/megatron/core/inference/model_inference_wrappers/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py b/megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/model_inference_wrappers/gpt/__init__.py b/megatron/core/inference/model_inference_wrappers/gpt/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py b/megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/model_inference_wrappers/inference_wrapper_config.py b/megatron/core/inference/model_inference_wrappers/inference_wrapper_config.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/model_inference_wrappers/t5/__init__.py b/megatron/core/inference/model_inference_wrappers/t5/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/model_inference_wrappers/t5/t5_inference_wrapper.py b/megatron/core/inference/model_inference_wrappers/t5/t5_inference_wrapper.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/modelopt_support/__init__.py b/megatron/core/inference/modelopt_support/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/modelopt_support/gpt/__init__.py b/megatron/core/inference/modelopt_support/gpt/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/modelopt_support/gpt/model_specs.py b/megatron/core/inference/modelopt_support/gpt/model_specs.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/modelopt_support/gpt/state_dict_hooks.py b/megatron/core/inference/modelopt_support/gpt/state_dict_hooks.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/sampling_params.py b/megatron/core/inference/sampling_params.py new file mode 100644 index 0000000..8ffcb63 --- /dev/null +++ b/megatron/core/inference/sampling_params.py @@ -0,0 +1,35 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +from dataclasses import dataclass + + +@dataclass +class SamplingParams: + """Inference parameters sent along with the prompts. + This class contains request-level attributes that control the sampling techniques used when + generating text. This is distinct from megatron.core.InferenceParams, which is sets model-level + inference attributes such as the maximum sequence length, and contains the KV cache. + + For an explanation of these parameters refer to this blog + https://ivibudh.medium.com/a-guide-to-controlling-llm-model-output-exploring-top-k-top-p-and- + temperature-parameters-ed6a31313910 + """ + + temperature: float = 1.0 + top_k: int = 0 + top_p: float = 0.0 + return_log_probs: bool = False + num_tokens_to_generate: int = 30 + + def add_attributes(self, attribute_value_pair: dict): + """Utility to add more attributes to sampling params + + Use this method to pass in a custom dictionary to add more sampling parameter attributes. + c = SamplingParams + c.add_attributes({'min_length':4, 'eod_id':153}) + + Args: + attribute_value_pair (dict): A dictionary containing attributes as the key names and + their values as the values. + """ + for key, value in attribute_value_pair.items(): + setattr(self, key, value) diff --git a/megatron/core/inference/scheduler.py b/megatron/core/inference/scheduler.py old mode 100755 new mode 100644 index 00ab81b..ef17723 --- a/megatron/core/inference/scheduler.py +++ b/megatron/core/inference/scheduler.py @@ -6,8 +6,8 @@ from typing import Dict import torch -from megatron.core.inference.common_inference_params import CommonInferenceParams from megatron.core.inference.inference_request import InferenceRequest, Status +from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import Counter @@ -33,7 +33,7 @@ class Scheduler: prompt: str, prompt_tokens: torch.Tensor, encoder_prompt: str = None, - inference_parameters: CommonInferenceParams = None, + inference_parameters: SamplingParams = None, arrival_time: float = None, ): """Add an incoming request @@ -45,7 +45,7 @@ class Scheduler: prompt (str): Input prompt string prompt_tokens (torch.Tensor): A torch tensor having the input prompts tokenized encoder_prompt (str): Encoder input string - inference_parameters (CommonInferenceParams): The inference parameters + inference_parameters (SamplingParams): The inference parameters arrival_time (float, optional): The incoming request time. Defaults to None. """ request_id = str(next(self.request_counter)) diff --git a/megatron/core/inference/text_generation_controllers/__init__.py b/megatron/core/inference/text_generation_controllers/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py b/megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py old mode 100755 new mode 100644 index 61beff0..0c2a41b --- a/megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py @@ -4,15 +4,15 @@ from typing import OrderedDict import torch from megatron.core.inference.inference_request import InferenceRequest -from megatron.core.inference.text_generation_controllers.simple_text_generation_controller import ( - SimpleTextGenerationController, +from megatron.core.inference.text_generation_controllers.text_generation_controller import ( + TextGenerationController, ) -class EncoderDecoderTextGenerationController(SimpleTextGenerationController): +class EncoderDecoderTextGenerationController(TextGenerationController): """The text generation controller for encoder-decoder architecture - This class ingherits from SimpleTextGenerationController, adding features + This class inherits from TextGenerationController, adding features relating to encoder input encoder_prompt """ diff --git a/megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py b/megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py old mode 100755 new mode 100644 index 1103089..f97df13 --- a/megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py @@ -1,400 +1,5 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -from typing import List, OrderedDict, Tuple +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -import torch -import torch.nn.functional as F - -from megatron.core import parallel_state -from megatron.core.inference.common_inference_params import CommonInferenceParams -from megatron.core.inference.communication_utils import broadcast_from_last_pipeline_stage -from megatron.core.inference.inference_request import InferenceRequest, Status -from megatron.core.inference.model_inference_wrappers.abstract_model_inference_wrapper import ( - AbstractModelInferenceWrapper, +from megatron.core.inference.text_generation_controllers.text_generation_controller import ( # noqa: F401 # pylint: disable=unused-import + TextGenerationController as SimpleTextGenerationController, ) - - -class SimpleTextGenerationController: - """The basic text generation controller - - This class is responsible for tokenizing the input , running the inference, sampling - and also detokenizing the output - - Args: - inference_wrapped_model (AbstractModelInferenceWrapper): A model that - is wrapped using the specs given in the abstract_model_inference_wrapper.py - tokenizer (_type_): Tokenizer used for tokenizing and detokenizing the prompts - """ - - def __init__(self, inference_wrapped_model: AbstractModelInferenceWrapper, tokenizer): - self.inference_wrapped_model = inference_wrapped_model - self.tokenizer = tokenizer - - # For models without pipeline parallelism, is_first_stage and is_last_stage returns True - self.model_is_pipeline_parallel = not ( - parallel_state.is_pipeline_first_stage() and parallel_state.is_pipeline_last_stage() - ) - - def tokenize_prompt( - self, prompt: str, add_BOS: bool = False - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Utility to tokenize the input prompts - - Args: - prompt (str): The input prompt - - Returns: - torch.Tensor: Returns the tokenized prompt - """ - prompt_tokens = self.tokenizer.tokenize(prompt) - - if add_BOS: - prompt_tokens = [self.tokenizer.bos] + prompt_tokens - - return prompt_tokens - - def detokenize_generations(self, prompt_tokens_with_generated_tokens: torch.Tensor) -> str: - """Detokenize the output generations - - Args: - prompt_tokens_with_generated_tokens (torch.Tensor): The input prompt - tokens plus the generated tokens - - Returns: - str: The detokenized output - """ - tokens = prompt_tokens_with_generated_tokens.cpu().numpy().tolist() - return self.tokenizer.detokenize(tokens) - - def sample_from_logits( - self, - last_token_logits: torch.Tensor, - common_inference_params: CommonInferenceParams, - vocab_size: int = None, - ) -> torch.Tensor: - """Samples the logits to generate outputs - - Given the logits of the last token, this function samples it - according to the parameters defined in common_inference_params - and returns the samples - - Args: - last_token_logits (torch.Tensor): The last token logits. A tensor of - size [batch_size, vocab_size] - common_inference_params (CommonInferenceParams): The paramters to use - for inference - vocab_size (int): Obtained from the tokenizer. Defaults to None - - Returns: - torch.Tensor: 1D tensor of the sampled logits with [batch_size] elements - """ - - top_p = common_inference_params.top_p - top_k = common_inference_params.top_k - temperature = common_inference_params.temperature - - assert not (top_k > 0 and top_p > 0), 'Cannot have top-p and top-k both greater than zero' - assert top_p <= 1.0, 'top-p should be in (0,1]' - - def modify_logits_for_top_k_filtering(logits, top_k): - """Set the logits for none top-k values to -inf.""" - filter_ = logits < torch.topk(logits, top_k)[0][..., -1, None] - logits.masked_fill_(filter_, float('-Inf')) - - def modify_logits_for_top_p_filtering(logits, top_p): - """Set the logits for none top-p values to -inf.""" - # First sort and calculate cumulative sum of probabilities. - sorted_logits, sorted_indices = torch.sort(logits, descending=True) - cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) - - # Filteration based on the cumulative sum. - filter_ = cumulative_probs > top_p - # This shift by 1 is weird and I cannot justify it. This existed - # in the original implementation: - # https://github.com/ari-holtzman/degen/blob/master/gen.py - # and I guess it is needed so keeping it for now. - filter_[:, 1:] = filter_[:, :-1].clone() - # Make sure we at least have one token to select from. - filter_[..., 0] = 0 - - # Fill in the filtered part - filter_ = filter_.scatter(1, sorted_indices, filter_) - logits.masked_fill_(filter_, float('-Inf')) - - # Greedy sampling - if top_k == 1: - sampled_logits = torch.argmax(last_token_logits, dim=-1) - else: - last_token_logits = last_token_logits.clone() - if temperature != 1.0: - last_token_logits.div_(temperature) - - if top_k > 1: - assert top_k <= last_token_logits.size(1), 'top-k is larger than logit size.' - if vocab_size: - assert top_k < vocab_size, 'top-k is larger than vocab size.' - modify_logits_for_top_k_filtering(last_token_logits, top_k) - - elif top_p > 0.0: - modify_logits_for_top_p_filtering(last_token_logits, top_p) - - # After filtering, we need to recalculate the distribution. - probabilities = last_token_logits.softmax(dim=-1) - sampled_logits = torch.multinomial(probabilities, num_samples=1).view(-1) - - # If vocab size is provided, make sure the samples are in in the range [0, vocab-size). - if vocab_size: - sampled_logits = torch.clamp(sampled_logits, min=0, max=(vocab_size - 1)) - return sampled_logits - - def update_generation_status( - self, - updated_prompts_tokens: torch.Tensor, - generation_started: torch.Tensor, - current_context_end_position: int, - is_generation_done_tensor: torch.Tensor, - generated_sequence_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Checks which prompts have reached an end condition - - We check which prompts have reached an end condition and set the corresponding - flags of the is_generation_done_tensor to True. The generated sequence lengths - increase as we keep generating, until that prompts hits an end condition. The - generation_started tensor determines which prompts have started generating. - - Args: - updated_prompts_tokens (torch.Tensor): The prompts tokens updated with the latest - generated tokens. A tensor of shape [batch_size, max_seq_len] - (i.e max_seq_len = max_prompt_len + tokens_to_generate) - generation_started (torch.Tensor): A boolean tensor of shape [batch_size]. True - indicates the prompt at that index has started generating tokens. - current_context_end_position (int): An integer indicating which position to - extract from the prompts tokens to get the latest generated tokens. - is_generation_done_tensor (torch.Tensor): A boolean tensor of shape [batch_size]. - True indicates the prompt at that index has reached end condition. - generated_sequence_lengths (torch.Tensor): A int tensor of shape [batch_size]. - Each value represents the generated sequence lengths for that prompt. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: Returns the boolean - is_generation_done_tensor and the generated_sequence_lengths after updating it - """ - latest_samples = updated_prompts_tokens[:, current_context_end_position] - # Make sure we are checking eod criterion only for prompts that have started generating - # (i.e) We only look at the generated tokenns and not the input tokens. - reached_eod = (latest_samples == self.tokenizer.eod) & generation_started - is_generation_done_tensor = is_generation_done_tensor | reached_eod - # We increment generated sequence lengths when that prompt has not hit the - # EOD and generation has started - generated_sequence_lengths += ~is_generation_done_tensor & generation_started - - return is_generation_done_tensor, generated_sequence_lengths - - def pad_input_prompt_tokens( - self, - batch_prompt_tokens_list: List[List[int]], - max_prompt_length_in_batch: int, - num_tokens_to_generate: int, - ) -> torch.Tensor: - """Method to pad input prompts - - Given a list of prompts, pad them all to uniform length - - Args: - batch_prompt_tokens_list (List[List[int]]): A list containing the prompt tokens - max_prompt_length_in_batch (int): Maximum of the length of the input prompt tokens - num_tokens_togenerate (int): The number of tokens to generate for each prompt - - Returns: - torch.Tensor: A torch tensor of shape [bs, max_seq_len] (i.e) - max_seq_len = max_prompt_length_in_batch + num_tokens_to_generate, - with extra indices for each tensor padded with mask id. - """ - max_seq_len = max_prompt_length_in_batch + num_tokens_to_generate - - for prompt_tokens in batch_prompt_tokens_list: - padding_size = max_seq_len - len(prompt_tokens) - prompt_tokens.extend([self.tokenizer.eod] * padding_size) - - return torch.tensor(batch_prompt_tokens_list).cuda() - - def generate_output_tokens_dynamic_batch( - self, active_requests: OrderedDict[int, InferenceRequest] - ) -> OrderedDict[int, InferenceRequest]: - """Utility to generate the output tokens and probabilities for the prompts - - This utility generates the output tokens for a dynamic batch. It will run one forward step - at a time, and pass control back to the engine, which will update the request pool and call - this method again. - - Args: - active_requests (OrderedDict[int, InferenceRequest]): The input active requests. - - Returns: - OrderedDict[int, InferenceRequest]: The result for each of the incoming requests - after running one forward step. - """ - raise Exception("Not implemented yet") - - def generate_all_output_tokens_static_batch( - self, active_requests: OrderedDict[int, InferenceRequest] - ) -> OrderedDict[int, InferenceRequest]: - """Utility to generate the all the output tokens and probabilities for the prompts . - - This utility generates the output tokens for a static batch. It runs the forward steps till - all prompts complete generation, updates the status of these requests to completed, adds - the generated result and returns these requests - - Args: - active_requests (OrderedDict[int, InferenceRequest]): The input active requests. - - Returns: - OrderedDict[int, InferenceRequest]: The result for each of the incoming requests - """ - batch_prompt_tokens_list = list( - map(lambda request: request.prompt_tokens, active_requests.values()) - ) - prompt_lengths_in_batch = torch.tensor( - [len(prompt_tokens) for prompt_tokens in batch_prompt_tokens_list] - ).cuda() - max_prompt_length_in_batch = max(prompt_lengths_in_batch) - min_prompt_length_in_batch = min(prompt_lengths_in_batch) - - # For batch inference the inference params are the same for all request - common_inference_params: CommonInferenceParams = list(active_requests.values())[ - 0 - ].inference_parameters - - # max_seq_len = max_prompt_length_in_batch + num_tokens_to_generate - batch_prompt_tokens = self.pad_input_prompt_tokens( - batch_prompt_tokens_list, - max_prompt_length_in_batch=max_prompt_length_in_batch, - num_tokens_to_generate=common_inference_params.num_tokens_to_generate, - ) - batch_size, max_sequence_length = batch_prompt_tokens.shape - - # Pre allocate log probs tensor - output_log_probs = None - if common_inference_params.return_log_probs: - output_log_probs = torch.empty( - (batch_size, max_sequence_length - 1), dtype=torch.float32 - ).cuda() - - # An array to check which of the prompts have reached end of generation condition - is_generation_done_tensor = torch.zeros(batch_size, dtype=torch.bool).cuda() - - # An array to act as a counter to keep track of generated sequence lengths - generated_sequence_lengths = torch.zeros(batch_size).cuda() - - with torch.no_grad(): - - self.prep_model_for_inference( - prompts_tokens=batch_prompt_tokens, active_requests=active_requests - ) - - context_start_position = 0 - # Pick the context window that we need to pass through the network. - for context_end_position in range(min_prompt_length_in_batch, max_sequence_length): - - inference_input = self.inference_wrapped_model.get_batch_for_context_window( - context_start_position, context_end_position - ) - - # Returns the final logits of shape [batch_size, context_length, vocab_size] - # Note: This is returned in all TP ranks or last PP stage in PP models - logits = self.inference_wrapped_model.run_one_forward_step(inference_input) - if self.model_is_pipeline_parallel: - context_length = context_end_position - context_start_position - logits = broadcast_from_last_pipeline_stage( - [batch_size, context_length, self.tokenizer.vocab_size], - dtype=self.inference_wrapped_model.inference_wrapper_config.params_dtype, - tensor=logits, - ) - - # Indicates which of the input prompts have started generating tokens. - # A 1D boolean tensor with [batch_size] elements (i.e) The shortest - # prompts will start generating first and so on - generation_started = prompt_lengths_in_batch <= context_end_position - last_token_logits = logits[:, -1, :] - sampled_logits = self.sample_from_logits( - last_token_logits, common_inference_params, self.tokenizer.vocab_size - ) - - # Substitute the sampled logits only for only the prompts that - # have started generating tokens - batch_prompt_tokens[generation_started, context_end_position] = sampled_logits[ - generation_started - ] - - if common_inference_params.return_log_probs: - log_probs = F.log_softmax(logits, dim=2) - indices = torch.unsqueeze( - batch_prompt_tokens[ - :, (context_start_position + 1) : (context_end_position + 1) - ], - 2, - ) - # Get the log probabilities for only the prompt tokens - output_log_probs[:, context_start_position:context_end_position] = torch.gather( - log_probs, 2, indices - ).squeeze(2) - - context_start_position = context_end_position - - # Check end of generation status for each tensor - # and update generated sequence lengths - (is_generation_done_tensor, generated_sequence_lengths) = ( - self.update_generation_status( - updated_prompts_tokens=batch_prompt_tokens, - generation_started=generation_started, - current_context_end_position=context_end_position, - is_generation_done_tensor=is_generation_done_tensor, - generated_sequence_lengths=generated_sequence_lengths, - ) - ) - # Boolean flag indicating if all prompts are finished - all_prompts_done = torch.all(is_generation_done_tensor) - if all_prompts_done: - break - - # Include all the generated tokens - batch_prompt_tokens_with_generations = batch_prompt_tokens[:, : (context_end_position + 1)] - if common_inference_params.return_log_probs: - output_log_probs = output_log_probs[:, :context_end_position] - - generated_sequence_lengths[ - generated_sequence_lengths > common_inference_params.num_tokens_to_generate - ] = common_inference_params.num_tokens_to_generate - - for idx, request in enumerate(active_requests.values()): - input_prompt_length = int(prompt_lengths_in_batch[idx]) - # Shorter prompts might have generated more than required tokens. So we trim them down - required_sequence_length = int( - min(generated_sequence_lengths[idx], common_inference_params.num_tokens_to_generate) - ) - # Extract only the generated tokens - required_result_tokens = batch_prompt_tokens_with_generations[ - idx, input_prompt_length : (input_prompt_length + required_sequence_length) - ] - - request.generated_length = required_sequence_length - request.generated_tokens = required_result_tokens - request.generated_log_probs = ( - None - if output_log_probs is None - else output_log_probs[idx, input_prompt_length:required_sequence_length] - ) - request.status = Status.COMPLETED - request.generated_text = self.detokenize_generations(required_result_tokens) - - return active_requests - - def prep_model_for_inference( - self, prompts_tokens: torch.Tensor, active_requests: OrderedDict[int, InferenceRequest] - ): - """Preparing batch for inference, using respective wrapper's prep_model_for_inference method - - Args: - prompts_tokens (torch.Tensor): A tensor of shape [batch_size, max_sequence_length] - active_requests (OrderedDict[int, InferenceRequest]): The input active requests - """ - self.inference_wrapped_model.prep_model_for_inference(prompts_tokens=prompts_tokens) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py new file mode 100644 index 0000000..f15c819 --- /dev/null +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -0,0 +1,400 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +from typing import List, OrderedDict, Tuple + +import torch +import torch.nn.functional as F + +from megatron.core import parallel_state +from megatron.core.inference.communication_utils import broadcast_from_last_pipeline_stage +from megatron.core.inference.inference_request import InferenceRequest, Status +from megatron.core.inference.model_inference_wrappers.abstract_model_inference_wrapper import ( + AbstractModelInferenceWrapper, +) +from megatron.core.inference.sampling_params import SamplingParams + + +class TextGenerationController: + """The text generation controller (the main sampling loop) + + This class tokenizes the input, runs inference, samples from logits, and detokenizes the output. + + Args: + inference_wrapped_model (AbstractModelInferenceWrapper): A model that + is wrapped using the specs given in the abstract_model_inference_wrapper.py + tokenizer (_type_): Tokenizer used for tokenizing and detokenizing the prompts + """ + + def __init__(self, inference_wrapped_model: AbstractModelInferenceWrapper, tokenizer): + self.inference_wrapped_model = inference_wrapped_model + self.tokenizer = tokenizer + + # For models without pipeline parallelism, is_first_stage and is_last_stage returns True + self.model_is_pipeline_parallel = not ( + parallel_state.is_pipeline_first_stage() and parallel_state.is_pipeline_last_stage() + ) + + def tokenize_prompt( + self, prompt: str, add_BOS: bool = False + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Utility to tokenize the input prompts + + Args: + prompt (str): The input prompt + + Returns: + torch.Tensor: Returns the tokenized prompt + """ + prompt_tokens = self.tokenizer.tokenize(prompt) + + if add_BOS: + prompt_tokens = [self.tokenizer.bos] + prompt_tokens + + return prompt_tokens + + def detokenize_generations(self, prompt_tokens_with_generated_tokens: torch.Tensor) -> str: + """Detokenize the output generations + + Args: + prompt_tokens_with_generated_tokens (torch.Tensor): The input prompt + tokens plus the generated tokens + + Returns: + str: The detokenized output + """ + tokens = prompt_tokens_with_generated_tokens.cpu().numpy().tolist() + return self.tokenizer.detokenize(tokens) + + def sample_from_logits( + self, + last_token_logits: torch.Tensor, + sampling_params: SamplingParams = None, + vocab_size: int = None, + **kwargs + ) -> torch.Tensor: + """Samples the logits to generate outputs + + Given the logits of the last token, this function samples it + according to the parameters defined in sampling_params + and returns the samples + + Args: + last_token_logits (torch.Tensor): The last token logits. A tensor of + size [batch_size, vocab_size] + sampling_params (SamplingParams): The parameters to use for inference. + vocab_size (int): Obtained from the tokenizer. Defaults to None + + Returns: + torch.Tensor: 1D tensor of the sampled logits with [batch_size] elements + """ + + if kwargs.get('common_inference_params'): + sampling_params = kwargs['common_inference_params'] + + top_p = sampling_params.top_p + top_k = sampling_params.top_k + temperature = sampling_params.temperature + + assert not (top_k > 0 and top_p > 0), 'Cannot have top-p and top-k both greater than zero' + assert top_p <= 1.0, 'top-p should be in (0,1]' + + def modify_logits_for_top_k_filtering(logits, top_k): + """Set the logits for none top-k values to -inf.""" + filter_ = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits.masked_fill_(filter_, float('-Inf')) + + def modify_logits_for_top_p_filtering(logits, top_p): + """Set the logits for none top-p values to -inf.""" + # First sort and calculate cumulative sum of probabilities. + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Filteration based on the cumulative sum. + filter_ = cumulative_probs > top_p + # This shift by 1 is weird and I cannot justify it. This existed + # in the original implementation: + # https://github.com/ari-holtzman/degen/blob/master/gen.py + # and I guess it is needed so keeping it for now. + filter_[:, 1:] = filter_[:, :-1].clone() + # Make sure we at least have one token to select from. + filter_[..., 0] = 0 + + # Fill in the filtered part + filter_ = filter_.scatter(1, sorted_indices, filter_) + logits.masked_fill_(filter_, float('-Inf')) + + # Greedy sampling + if top_k == 1: + sampled_logits = torch.argmax(last_token_logits, dim=-1) + else: + last_token_logits = last_token_logits.clone() + if temperature != 1.0: + last_token_logits.div_(temperature) + + if top_k > 1: + assert top_k <= last_token_logits.size(1), 'top-k is larger than logit size.' + if vocab_size: + assert top_k < vocab_size, 'top-k is larger than vocab size.' + modify_logits_for_top_k_filtering(last_token_logits, top_k) + + elif top_p > 0.0: + modify_logits_for_top_p_filtering(last_token_logits, top_p) + + # After filtering, we need to recalculate the distribution. + probabilities = last_token_logits.softmax(dim=-1) + sampled_logits = torch.multinomial(probabilities, num_samples=1).view(-1) + + # If vocab size is provided, make sure the samples are in in the range [0, vocab-size). + if vocab_size: + sampled_logits = torch.clamp(sampled_logits, min=0, max=(vocab_size - 1)) + return sampled_logits + + def update_generation_status( + self, + updated_prompts_tokens: torch.Tensor, + generation_started: torch.Tensor, + current_context_end_position: int, + is_generation_done_tensor: torch.Tensor, + generated_sequence_lengths: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Checks which prompts have reached an end condition + + We check which prompts have reached an end condition and set the corresponding + flags of the is_generation_done_tensor to True. The generated sequence lengths + increase as we keep generating, until that prompts hits an end condition. The + generation_started tensor determines which prompts have started generating. + + Args: + updated_prompts_tokens (torch.Tensor): The prompts tokens updated with the latest + generated tokens. A tensor of shape [batch_size, max_seq_len] + (i.e max_seq_len = max_prompt_len + tokens_to_generate) + generation_started (torch.Tensor): A boolean tensor of shape [batch_size]. True + indicates the prompt at that index has started generating tokens. + current_context_end_position (int): An integer indicating which position to + extract from the prompts tokens to get the latest generated tokens. + is_generation_done_tensor (torch.Tensor): A boolean tensor of shape [batch_size]. + True indicates the prompt at that index has reached end condition. + generated_sequence_lengths (torch.Tensor): A int tensor of shape [batch_size]. + Each value represents the generated sequence lengths for that prompt. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: Returns the boolean + is_generation_done_tensor and the generated_sequence_lengths after updating it + """ + latest_samples = updated_prompts_tokens[:, current_context_end_position] + # Make sure we are checking eod criterion only for prompts that have started generating + # (i.e) We only look at the generated tokenns and not the input tokens. + reached_eod = (latest_samples == self.tokenizer.eod) & generation_started + is_generation_done_tensor = is_generation_done_tensor | reached_eod + # We increment generated sequence lengths when that prompt has not hit the + # EOD and generation has started + generated_sequence_lengths += ~is_generation_done_tensor & generation_started + + return is_generation_done_tensor, generated_sequence_lengths + + def pad_input_prompt_tokens( + self, + batch_prompt_tokens_list: List[List[int]], + max_prompt_length_in_batch: int, + num_tokens_to_generate: int, + ) -> torch.Tensor: + """Method to pad input prompts + + Given a list of prompts, pad them all to uniform length + + Args: + batch_prompt_tokens_list (List[List[int]]): A list containing the prompt tokens + max_prompt_length_in_batch (int): Maximum of the length of the input prompt tokens + num_tokens_togenerate (int): The number of tokens to generate for each prompt + + Returns: + torch.Tensor: A torch tensor of shape [bs, max_seq_len] (i.e) + max_seq_len = max_prompt_length_in_batch + num_tokens_to_generate, + with extra indices for each tensor padded with mask id. + """ + max_seq_len = max_prompt_length_in_batch + num_tokens_to_generate + + for prompt_tokens in batch_prompt_tokens_list: + padding_size = max_seq_len - len(prompt_tokens) + prompt_tokens.extend([self.tokenizer.eod] * padding_size) + + return torch.tensor(batch_prompt_tokens_list).cuda() + + def generate_output_tokens_dynamic_batch( + self, active_requests: OrderedDict[int, InferenceRequest] + ) -> OrderedDict[int, InferenceRequest]: + """Utility to generate the output tokens and probabilities for the prompts + + This utility generates the output tokens for a dynamic batch. It will run one forward step + at a time, and pass control back to the engine, which will update the request pool and call + this method again. + + Args: + active_requests (OrderedDict[int, InferenceRequest]): The input active requests. + + Returns: + OrderedDict[int, InferenceRequest]: The result for each of the incoming requests + after running one forward step. + """ + raise Exception("Not implemented yet") + + def generate_all_output_tokens_static_batch( + self, active_requests: OrderedDict[int, InferenceRequest] + ) -> OrderedDict[int, InferenceRequest]: + """Utility to generate the all the output tokens and probabilities for the prompts . + + This utility generates the output tokens for a static batch. It runs the forward steps till + all prompts complete generation, updates the status of these requests to completed, adds + the generated result and returns these requests + + Args: + active_requests (OrderedDict[int, InferenceRequest]): The input active requests. + + Returns: + OrderedDict[int, InferenceRequest]: The result for each of the incoming requests + """ + batch_prompt_tokens_list = list( + map(lambda request: request.prompt_tokens, active_requests.values()) + ) + prompt_lengths_in_batch = torch.tensor( + [len(prompt_tokens) for prompt_tokens in batch_prompt_tokens_list] + ).cuda() + max_prompt_length_in_batch = max(prompt_lengths_in_batch) + min_prompt_length_in_batch = min(prompt_lengths_in_batch) + + # For batch inference the inference params are the same for all request + sampling_params: SamplingParams = list(active_requests.values())[0].inference_parameters + + # max_seq_len = max_prompt_length_in_batch + num_tokens_to_generate + batch_prompt_tokens = self.pad_input_prompt_tokens( + batch_prompt_tokens_list, + max_prompt_length_in_batch=max_prompt_length_in_batch, + num_tokens_to_generate=sampling_params.num_tokens_to_generate, + ) + batch_size, max_sequence_length = batch_prompt_tokens.shape + + # Pre allocate log probs tensor + output_log_probs = None + if sampling_params.return_log_probs: + output_log_probs = torch.empty( + (batch_size, max_sequence_length - 1), dtype=torch.float32 + ).cuda() + + # An array to check which of the prompts have reached end of generation condition + is_generation_done_tensor = torch.zeros(batch_size, dtype=torch.bool).cuda() + + # An array to act as a counter to keep track of generated sequence lengths + generated_sequence_lengths = torch.zeros(batch_size).cuda() + + with torch.no_grad(): + + self.prep_model_for_inference( + prompts_tokens=batch_prompt_tokens, active_requests=active_requests + ) + + context_start_position = 0 + # Pick the context window that we need to pass through the network. + for context_end_position in range(min_prompt_length_in_batch, max_sequence_length): + + inference_input = self.inference_wrapped_model.get_batch_for_context_window( + context_start_position, context_end_position + ) + + # Returns the final logits of shape [batch_size, context_length, vocab_size] + # Note: This is returned in all TP ranks or last PP stage in PP models + logits = self.inference_wrapped_model.run_one_forward_step(inference_input) + if self.model_is_pipeline_parallel: + context_length = context_end_position - context_start_position + logits = broadcast_from_last_pipeline_stage( + [batch_size, context_length, self.tokenizer.vocab_size], + dtype=self.inference_wrapped_model.inference_wrapper_config.params_dtype, + tensor=logits, + ) + + # Indicates which of the input prompts have started generating tokens. + # A 1D boolean tensor with [batch_size] elements (i.e) The shortest + # prompts will start generating first and so on + generation_started = prompt_lengths_in_batch <= context_end_position + last_token_logits = logits[:, -1, :] + sampled_logits = self.sample_from_logits( + last_token_logits, sampling_params, self.tokenizer.vocab_size + ) + + # Substitute the sampled logits only for only the prompts that + # have started generating tokens + batch_prompt_tokens[generation_started, context_end_position] = sampled_logits[ + generation_started + ] + + if sampling_params.return_log_probs: + log_probs = F.log_softmax(logits, dim=2) + indices = torch.unsqueeze( + batch_prompt_tokens[ + :, (context_start_position + 1) : (context_end_position + 1) + ], + 2, + ) + # Get the log probabilities for only the prompt tokens + output_log_probs[:, context_start_position:context_end_position] = torch.gather( + log_probs, 2, indices + ).squeeze(2) + + context_start_position = context_end_position + + # Check end of generation status for each tensor + # and update generated sequence lengths + (is_generation_done_tensor, generated_sequence_lengths) = ( + self.update_generation_status( + updated_prompts_tokens=batch_prompt_tokens, + generation_started=generation_started, + current_context_end_position=context_end_position, + is_generation_done_tensor=is_generation_done_tensor, + generated_sequence_lengths=generated_sequence_lengths, + ) + ) + # Boolean flag indicating if all prompts are finished + all_prompts_done = torch.all(is_generation_done_tensor) + if all_prompts_done: + break + + # Include all the generated tokens + batch_prompt_tokens_with_generations = batch_prompt_tokens[:, : (context_end_position + 1)] + if sampling_params.return_log_probs: + output_log_probs = output_log_probs[:, :context_end_position] + + generated_sequence_lengths[ + generated_sequence_lengths > sampling_params.num_tokens_to_generate + ] = sampling_params.num_tokens_to_generate + + for idx, request in enumerate(active_requests.values()): + input_prompt_length = int(prompt_lengths_in_batch[idx]) + # Shorter prompts might have generated more than required tokens. So we trim them down + required_sequence_length = int( + min(generated_sequence_lengths[idx], sampling_params.num_tokens_to_generate) + ) + # Extract only the generated tokens + required_result_tokens = batch_prompt_tokens_with_generations[ + idx, input_prompt_length : (input_prompt_length + required_sequence_length) + ] + + request.generated_length = required_sequence_length + request.generated_tokens = required_result_tokens + request.generated_log_probs = ( + None + if output_log_probs is None + else output_log_probs[idx, input_prompt_length:required_sequence_length] + ) + request.status = Status.COMPLETED + request.generated_text = self.detokenize_generations(required_result_tokens) + + return active_requests + + def prep_model_for_inference( + self, prompts_tokens: torch.Tensor, active_requests: OrderedDict[int, InferenceRequest] + ): + """Preparing batch for inference, using respective wrapper's prep_model_for_inference method + + Args: + prompts_tokens (torch.Tensor): A tensor of shape [batch_size, max_sequence_length] + active_requests (OrderedDict[int, InferenceRequest]): The input active requests + """ + self.inference_wrapped_model.prep_model_for_inference(prompts_tokens=prompts_tokens) diff --git a/megatron/core/inference/utils.py b/megatron/core/inference/utils.py old mode 100755 new mode 100644 diff --git a/megatron/core/inference_params.py b/megatron/core/inference_params.py old mode 100755 new mode 100644 diff --git a/megatron/core/jit.py b/megatron/core/jit.py old mode 100755 new mode 100644 index 5b1dfff..c35c41b --- a/megatron/core/jit.py +++ b/megatron/core/jit.py @@ -7,4 +7,18 @@ from megatron.core.utils import is_torch_min_version jit_fuser = torch.jit.script # nvFuser is deprecated in PyTorch JIT starting from 2.2 if is_torch_min_version("2.2.0a0"): - jit_fuser = torch.compile + jit_fuser = torch.compile(mode='max-autotune-no-cudagraphs') + +# Decorator to disable Torch Dynamo +# See: https://github.com/NVIDIA/TransformerEngine/issues/308 +no_torch_dynamo = lambda recursive=True: lambda func: func +if torch.__version__ >= "2": + import torch._dynamo + + if torch.__version__ >= "2.1": + no_torch_dynamo = lambda recursive=True: lambda f: torch._dynamo.disable( + f, recursive=recursive + ) + else: + # no "recursive" option in pyTorch 2.0 - it acts as if recursive was True + no_torch_dynamo = lambda recursive=True: torch._dynamo.disable diff --git a/megatron/core/model_parallel_config.py b/megatron/core/model_parallel_config.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/T5/__init__.py b/megatron/core/models/T5/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/T5/t5_model.py b/megatron/core/models/T5/t5_model.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/T5/t5_spec.py b/megatron/core/models/T5/t5_spec.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/__init__.py b/megatron/core/models/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/bert/__init__.py b/megatron/core/models/bert/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/bert/bert_layer_specs.py b/megatron/core/models/bert/bert_layer_specs.py old mode 100755 new mode 100644 index 80893d5..4edc2ed --- a/megatron/core/models/bert/bert_layer_specs.py +++ b/megatron/core/models/bert/bert_layer_specs.py @@ -1,4 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +import warnings + from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules @@ -28,38 +30,60 @@ try: HAVE_APEX = True LNImpl = FusedLayerNorm except ImportError: - import warnings from megatron.core.transformer.torch_norm import WrappedTorchNorm - warnings.warn(f'Apex is not installed. Falling back to Torch Norm') + warnings.warn('Apex is not installed. Falling back to Torch Norm') LNImpl = WrappedTorchNorm -# Use this spec to use lower level Transformer Engine modules (required for fp8 training) -bert_layer_with_transformer_engine_spec = ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - self_attention=ModuleSpec( - module=SelfAttention, - params={"attn_mask_type": AttnMaskType.padding}, - submodules=SelfAttentionSubmodules( - linear_qkv=TELayerNormColumnParallelLinear, - core_attention=TEDotProductAttention, - linear_proj=TERowParallelLinear, - q_layernorm=IdentityOp, - k_layernorm=IdentityOp, + +def get_bert_layer_with_transformer_engine_spec(): + """Use this spec to use lower-level Transformer Engine modules (required for fp8 training). + + Returns: + ModuleSpec: Module specification with TE modules + """ + if not HAVE_TE: + raise ImportError( + "Transformer Engine is not installed. Please use local Bert layer spec instead." + ) + + return ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.padding}, + submodules=SelfAttentionSubmodules( + linear_qkv=TELayerNormColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=IdentityOp, + k_layernorm=IdentityOp, + ), ), - ), - self_attn_bda=get_bias_dropout_add, - mlp=ModuleSpec( - module=MLP, - submodules=MLPSubmodules( - linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear + self_attn_bda=get_bias_dropout_add, + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear + ), ), + mlp_bda=get_bias_dropout_add, ), - mlp_bda=get_bias_dropout_add, - ), -) + ) + + +def __getattr__(name): + if name == 'bert_layer_with_transformer_engine_spec': + warnings.warn( + """Attribute bert_layer_specs.bert_layer_with_transformer_engine_spec is on a + deprecation track and will be removed in future releases. Please migrate to + bert_layer_specs.get_bert_layer_with_transformer_engine_spec().""" + ) + + return get_bert_layer_with_transformer_engine_spec() + # Use this spec for an implementation using only modules in megatron core bert_layer_local_spec = ModuleSpec( diff --git a/megatron/core/models/bert/bert_lm_head.py b/megatron/core/models/bert/bert_lm_head.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/bert/bert_model.py b/megatron/core/models/bert/bert_model.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/bert/pooler.py b/megatron/core/models/bert/pooler.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/__init__.py b/megatron/core/models/common/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/embeddings/__init__.py b/megatron/core/models/common/embeddings/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/embeddings/language_model_embedding.py b/megatron/core/models/common/embeddings/language_model_embedding.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/embeddings/rope_utils.py b/megatron/core/models/common/embeddings/rope_utils.py old mode 100755 new mode 100644 index f1d7ad4..3dd5193 --- a/megatron/core/models/common/embeddings/rope_utils.py +++ b/megatron/core/models/common/embeddings/rope_utils.py @@ -17,23 +17,24 @@ from megatron.core.utils import is_te_min_version logger = logging.getLogger(__name__) +# Prefer fused RoPE from Apex as we need the `transpose_output_memory` argument for the bshd trick. +# See https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/merge_requests/2469. try: - from megatron.core.extensions.transformer_engine import ( - fused_apply_rotary_pos_emb, - fused_apply_rotary_pos_emb_thd, - ) - - HAVE_APPLY_ROPE_FUSION = True + from apex.transformer.functional import fused_apply_rotary_pos_emb except ImportError: try: - from apex.transformer.functional import ( - fused_apply_rotary_pos_emb, - fused_apply_rotary_pos_emb_thd, - ) + from megatron.core.extensions.transformer_engine import fused_apply_rotary_pos_emb + except: + fused_apply_rotary_pos_emb = None + - HAVE_APPLY_ROPE_FUSION = True +try: + from megatron.core.extensions.transformer_engine import fused_apply_rotary_pos_emb_thd +except ImportError: + try: + from apex.transformer.functional import fused_apply_rotary_pos_emb_thd except ImportError: - HAVE_APPLY_ROPE_FUSION = False + fused_apply_rotary_pos_emb_thd = None try: @@ -188,8 +189,10 @@ def apply_rotary_pos_emb( if config.apply_rope_fusion: if cu_seqlens is None: - return fused_apply_rotary_pos_emb(t, freqs) + assert fused_apply_rotary_pos_emb is not None, "apply_rope_fusion is not available." + return fused_apply_rotary_pos_emb(t, freqs, transpose_output_memory=True) else: + assert fused_apply_rotary_pos_emb_thd is not None, "apply_rope_fusion is not available." cp_size = parallel_state.get_context_parallel_world_size() if cp_size > 1: if not is_te_min_version("1.11.0", check_equality=False): diff --git a/megatron/core/models/common/embeddings/rotary_pos_embedding.py b/megatron/core/models/common/embeddings/rotary_pos_embedding.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/language_module/__init__.py b/megatron/core/models/common/language_module/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/vision_module/__init__.py b/megatron/core/models/common/vision_module/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/common/vision_module/vision_module.py b/megatron/core/models/common/vision_module/vision_module.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/gpt/__init__.py b/megatron/core/models/gpt/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/gpt/gpt_layer_specs.py b/megatron/core/models/gpt/gpt_layer_specs.py old mode 100755 new mode 100644 index 749be32..d0e48c1 --- a/megatron/core/models/gpt/gpt_layer_specs.py +++ b/megatron/core/models/gpt/gpt_layer_specs.py @@ -1,16 +1,16 @@ # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +import warnings from typing import Optional from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add +from megatron.core.models.gpt.moe_module_specs import get_moe_module_spec from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules from megatron.core.transformer.dot_product_attention import DotProductAttention from megatron.core.transformer.enums import AttnMaskType from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.mlp import MLP, MLPSubmodules -from megatron.core.transformer.moe.moe_layer import MoELayer, MoESubmodules -from megatron.core.transformer.moe.shared_experts import SharedExpertMLP from megatron.core.transformer.multi_latent_attention import ( MLASelfAttention, MLASelfAttentionSubmodules, @@ -26,12 +26,10 @@ from megatron.core.utils import is_te_min_version try: from megatron.core.extensions.transformer_engine import ( - TEColumnParallelGroupedLinear, TEColumnParallelLinear, TEDotProductAttention, TELayerNormColumnParallelLinear, TENorm, - TERowParallelGroupedLinear, TERowParallelLinear, ) @@ -47,8 +45,6 @@ try: HAVE_APEX = True LNImpl = FusedLayerNorm except ImportError: - import warnings - from megatron.core.transformer.torch_norm import WrappedTorchNorm warnings.warn('Apex is not installed. Falling back to Torch Norm') @@ -60,7 +56,8 @@ def get_gpt_layer_with_transformer_engine_spec( moe_grouped_gemm: Optional[bool] = False, qk_layernorm: Optional[bool] = False, multi_latent_attention: Optional[bool] = False, - fp8: Optional[str] = None, + fp8: Optional[str] = None, # pylint: disable=unused-arguments + moe_use_legacy_grouped_gemm: Optional[bool] = False, ) -> ModuleSpec: """Use this spec to use lower-level Transformer Engine modules (required for fp8 training). @@ -69,13 +66,24 @@ def get_gpt_layer_with_transformer_engine_spec( num_experts (int, optional): Number of experts. Defaults to None. moe_grouped_gemm (bool, optional): To use Grouped GEMM. Defaults to False. qk_layernorm (bool, optional): To use layernorm for queries/keys. Defaults to False. - fp8 (str, optional): Flag to decide the linear layer spec for MoE. Defaults to None. + fp8 (str, optional): Deprecated. For temporary Nemo compatibility. + moe_use_legacy_grouped_gemm (bool, optional): Force use the legacy GroupedMLP. + Defaults to False. Returns: ModuleSpec: Module specification with TE modules """ + if fp8 is not None: + warnings.warn( + 'The fp8 argument in "get_gpt_layer_with_transformer_engine_spec" has been deprecated' + ' and will be removed soon. Please update your code accordingly.' + ) + mlp = _get_mlp_module_spec( - use_te=True, num_experts=num_experts, moe_grouped_gemm=moe_grouped_gemm, fp8=fp8 + use_te=True, + num_experts=num_experts, + moe_grouped_gemm=moe_grouped_gemm, + moe_use_legacy_grouped_gemm=moe_use_legacy_grouped_gemm, ) if multi_latent_attention: @@ -138,6 +146,8 @@ def get_gpt_layer_local_spec( moe_grouped_gemm: Optional[bool] = False, qk_layernorm: Optional[bool] = False, multi_latent_attention: Optional[bool] = False, + fp8: Optional[str] = None, # pylint: disable=unused-arguments + moe_use_legacy_grouped_gemm: Optional[bool] = False, ) -> ModuleSpec: """Use this spec for an implementation using only modules in Megatron-Core. @@ -146,13 +156,24 @@ def get_gpt_layer_local_spec( num_experts (int, optional): Number of experts. Defaults to None. moe_grouped_gemm (bool, optional): To use Grouped GEMM. Defaults to False. qk_layernorm (bool, optional): To use layernorm for queries/keys. Defaults to False. + fp8 (str, optional): Deprecated. For temporary Nemo compatibility. + moe_use_legacy_grouped_gemm (bool, optional): Force use the legacy GroupedMLP. + Defaults to False. Returns: ModuleSpec: Module specification with Megatron-Core modules """ + if fp8 is not None: + warnings.warn( + 'The fp8 argument in "get_gpt_layer_local_spec" has been deprecated' + ' and will be removed soon. Please update your code accordingly.' + ) mlp = _get_mlp_module_spec( - use_te=False, num_experts=num_experts, moe_grouped_gemm=moe_grouped_gemm + use_te=False, + num_experts=num_experts, + moe_grouped_gemm=moe_grouped_gemm, + moe_use_legacy_grouped_gemm=moe_use_legacy_grouped_gemm, ) if multi_latent_attention: @@ -213,63 +234,33 @@ def _get_mlp_module_spec( use_te: Optional[bool] = True, num_experts: Optional[int] = None, moe_grouped_gemm: Optional[bool] = False, - fp8: Optional[str] = None, + fp8: Optional[str] = None, # pylint: disable=unused-arguments + moe_use_legacy_grouped_gemm: Optional[bool] = False, ) -> ModuleSpec: - """Helper function to get module spec for MLP""" - if num_experts is not None: - moe_spec = _get_moe_module_spec( - use_te=True, num_experts=num_experts, moe_grouped_gemm=moe_grouped_gemm, fp8=fp8 + """Helper function to get module spec for MLP/MoE""" + if fp8 is not None: + warnings.warn( + 'The fp8 argument in "_get_mlp_module_spec" has been deprecated' + ' and will be removed soon. Please update your code accordingly.' ) - return moe_spec - - return ModuleSpec( - module=MLP, - submodules=MLPSubmodules( - linear_fc1=TELayerNormColumnParallelLinear if use_te else ColumnParallelLinear, - linear_fc2=TERowParallelLinear if use_te else RowParallelLinear, - ), - ) - -def _get_moe_module_spec( - use_te: Optional[bool] = True, - num_experts: Optional[int] = None, - moe_grouped_gemm: Optional[bool] = False, - fp8: Optional[str] = None, -) -> ModuleSpec: - """Helper function to get module spec for MoE""" if num_experts is None: - return None - if use_te and moe_grouped_gemm: - linear_fc1 = TEColumnParallelGroupedLinear - linear_fc2 = TERowParallelGroupedLinear - elif use_te and fp8: - linear_fc1 = TEColumnParallelLinear - linear_fc2 = TERowParallelLinear - else: - linear_fc1 = ColumnParallelLinear - linear_fc2 = RowParallelLinear - - use_te_grouped_gemm = use_te and TEColumnParallelGroupedLinear is not None - - return ModuleSpec( - module=MoELayer, - submodules=MoESubmodules( - experts=( - MLPSubmodules(linear_fc1=linear_fc1, linear_fc2=linear_fc2) - if not moe_grouped_gemm or use_te_grouped_gemm - else None - ), - shared_experts=ModuleSpec( - module=SharedExpertMLP, - params={"gate": False}, - submodules=MLPSubmodules( - linear_fc1=TEColumnParallelLinear if use_te else ColumnParallelLinear, - linear_fc2=TERowParallelLinear if use_te else RowParallelLinear, - ), + # Dense MLP w/ or w/o TE modules. + return ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TELayerNormColumnParallelLinear if use_te else ColumnParallelLinear, + linear_fc2=TERowParallelLinear if use_te else RowParallelLinear, ), - ), - ) + ) + else: + # Mixture of experts with modules in megatron core. + return get_moe_module_spec( + use_te=use_te, + num_experts=num_experts, + moe_grouped_gemm=moe_grouped_gemm, + moe_use_legacy_grouped_gemm=moe_use_legacy_grouped_gemm, + ) def get_gpt_decoder_block_spec( @@ -288,7 +279,7 @@ def get_gpt_decoder_block_spec( moe_grouped_gemm=False, qk_layernorm=config.qk_layernorm, multi_latent_attention=config.multi_latent_attention, - fp8=config.fp8, + moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, ) if use_transformer_engine else get_gpt_layer_local_spec( @@ -296,6 +287,7 @@ def get_gpt_decoder_block_spec( moe_grouped_gemm=False, qk_layernorm=config.qk_layernorm, multi_latent_attention=config.multi_latent_attention, + moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, ) ) moe_layer_spec = ( @@ -304,7 +296,7 @@ def get_gpt_decoder_block_spec( moe_grouped_gemm=config.moe_grouped_gemm, qk_layernorm=config.qk_layernorm, multi_latent_attention=config.multi_latent_attention, - fp8=config.fp8, + moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, ) if use_transformer_engine else get_gpt_layer_local_spec( @@ -312,6 +304,7 @@ def get_gpt_decoder_block_spec( moe_grouped_gemm=config.moe_grouped_gemm, qk_layernorm=config.qk_layernorm, multi_latent_attention=config.multi_latent_attention, + moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, ) ) diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/gpt/moe_module_specs.py b/megatron/core/models/gpt/moe_module_specs.py new file mode 100644 index 0000000..513eedd --- /dev/null +++ b/megatron/core/models/gpt/moe_module_specs.py @@ -0,0 +1,81 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + +import warnings +from typing import Optional + +from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear +from megatron.core.transformer.mlp import MLPSubmodules +from megatron.core.transformer.moe.experts import GroupedMLP, SequentialMLP, TEGroupedMLP +from megatron.core.transformer.moe.moe_layer import MoELayer, MoESubmodules +from megatron.core.transformer.moe.shared_experts import SharedExpertMLP +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.utils import get_te_version, is_te_min_version + +try: + from megatron.core.extensions.transformer_engine import ( + TEColumnParallelGroupedLinear, + TEColumnParallelLinear, + TERowParallelGroupedLinear, + TERowParallelLinear, + ) + + HAVE_TE = True +except ImportError: + HAVE_TE = False + + +def get_moe_module_spec( + use_te: Optional[bool] = True, + num_experts: Optional[int] = None, + moe_grouped_gemm: Optional[bool] = False, + moe_use_legacy_grouped_gemm: Optional[bool] = False, +) -> ModuleSpec: + """Helper function to get module spec for MoE""" + assert num_experts is not None + + mlp = MLPSubmodules( + linear_fc1=TEColumnParallelLinear if use_te else ColumnParallelLinear, + linear_fc2=TERowParallelLinear if use_te else RowParallelLinear, + ) + + # experts spec + if moe_grouped_gemm: + ## use GroupedMLP + if use_te and TEColumnParallelGroupedLinear is not None and not moe_use_legacy_grouped_gemm: + ## use TEGroupedLinear + expert_module = TEGroupedMLP + expert_submodule = MLPSubmodules( + linear_fc1=TEColumnParallelGroupedLinear, linear_fc2=TERowParallelGroupedLinear + ) + else: + ## use legacy GroupedMLP + expert_module = GroupedMLP + expert_submodule = None + warnings.warn( + 'The legacy GroupedMLP will be deprecated in Megatron-Core v0.12.0. ' + 'Please update the TransformerEngine to version>=1.7.0 and use TEGroupedMLP.' + ) + else: + ## use SequentialMLP + expert_module = SequentialMLP + if use_te and not is_te_min_version("1.7.0.dev0"): + warnings.warn( + "Only transformer-engine>=1.7.0 supports MoE experts, " + f"but your version is {get_te_version()}. Use local linear implementation instead." + ) + expert_submodule = MLPSubmodules( + linear_fc1=ColumnParallelLinear, linear_fc2=RowParallelLinear + ) + else: + expert_submodule = mlp + + experts = ModuleSpec(module=expert_module, submodules=expert_submodule) + + # shared experts spec + shared_experts = ModuleSpec(module=SharedExpertMLP, params={"gate": False}, submodules=mlp) + + # MoE module spec + moe_module_spec = ModuleSpec( + module=MoELayer, submodules=MoESubmodules(experts=experts, shared_experts=shared_experts) + ) + return moe_module_spec diff --git a/megatron/core/models/mamba/__init__.py b/megatron/core/models/mamba/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/mamba/mamba_layer_specs.py b/megatron/core/models/mamba/mamba_layer_specs.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/mamba/mamba_model.py b/megatron/core/models/mamba/mamba_model.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/multimodal/__init__.py b/megatron/core/models/multimodal/__init__.py old mode 100755 new mode 100644 diff --git a/megatron/core/models/multimodal/llava_model.py b/megatron/core/models/multimodal/llava_model.py old mode 100755 new mode 100644 index dafe377..3de68b5 --- a/megatron/core/models/multimodal/llava_model.py +++ b/megatron/core/models/multimodal/llava_model.py @@ -36,6 +36,7 @@ IGNORE_INDEX = -100 # ID for labels that should be ignored. # Image token index can be tokenizer dependent so the default value does not work in all cases. DEFAULT_IMAGE_TOKEN_INDEX = -200 IMAGE_TOKEN = "" +VIDEO_TOKEN = "