First add

d3dd8642 · Rayyyyy · d3dd8642 · d3dd8642 · d3dd8642 · d3dd8642
Commit d3dd8642 authored Jun 26, 2024 by Rayyyyy
20 changed files
--- a/megatron/data/biencoder_dataset_utils.py
+++ b/megatron/data/biencoder_dataset_utils.py
--- a/megatron/data/blendable_dataset.py
+++ b/megatron/data/blendable_dataset.py
--- a/megatron/data/data_samplers.py
+++ b/megatron/data/data_samplers.py
--- a/megatron/data/dataset_utils.py
+++ b/megatron/data/dataset_utils.py
--- a/megatron/data/gpt_dataset.py
+++ b/megatron/data/gpt_dataset.py
--- a/megatron/data/helpers.cpp
+++ b/megatron/data/helpers.cpp
--- a/megatron/data/helpers.cpython-310-x86_64-linux-gnu.so
+++ b/megatron/data/helpers.cpython-310-x86_64-linux-gnu.so
--- a/megatron/data/ict_dataset.py
+++ b/megatron/data/ict_dataset.py
--- a/megatron/data/image_folder.py
+++ b/megatron/data/image_folder.py
--- a/megatron/data/indexed_dataset.py
+++ b/megatron/data/indexed_dataset.py
--- a/megatron/data/multimodal_dataset.py
+++ b/megatron/data/multimodal_dataset.py
--- a/megatron/data/orqa_wiki_dataset.py
+++ b/megatron/data/orqa_wiki_dataset.py
--- a/megatron/data/realm_dataset_utils.py
+++ b/megatron/data/realm_dataset_utils.py
--- a/megatron/data/realm_index.py
+++ b/megatron/data/realm_index.py
--- a/megatron/data/t5_dataset.py
+++ b/megatron/data/t5_dataset.py
--- a/megatron/data/test/test_indexed_dataset.py
+++ b/megatron/data/test/test_indexed_dataset.py
--- a/megatron/data/test/test_preprocess_data.sh
+++ b/megatron/data/test/test_preprocess_data.sh
--- a/megatron/data/vit_dataset.py
+++ b/megatron/data/vit_dataset.py
--- a/megatron/dist_signal_handler.py
+++ b/megatron/dist_signal_handler.py
--- a/megatron/fp16_deprecated/loss_scaler.py
+++ b/megatron/fp16_deprecated/loss_scaler.py