Initial commit

deb8370c · hepj · deb8370c · deb8370c · deb8370c · deb8370c
Commit deb8370c authored Jan 09, 2025 by hepj
20 changed files
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/factory.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/factory.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/index.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/index.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/indexes/__init__.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/indexes/__init__.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/indexes/faiss_base.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/indexes/faiss_base.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/indexes/faiss_par_add.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/indexes/faiss_par_add.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/utils.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/utils.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/validate.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/index/validate.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/__init__.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/__init__.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/gpt_chunk_dataset.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/gpt_chunk_dataset.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/multi_split_gpt_dataset.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/multi_split_gpt_dataset.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/query.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/query.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/retro_dataset.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/retro_dataset.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/utils.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/query/utils.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/retro/utils.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/retro/utils.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/t5_dataset.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/t5_dataset.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/utils.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/utils.py
--- a/PAI-Megatron-LM-240718/megatron/core/datasets/utils_s3.py
+++ b/PAI-Megatron-LM-240718/megatron/core/datasets/utils_s3.py
--- a/PAI-Megatron-LM-240718/megatron/core/dist_checkpointing/__init__.py
+++ b/PAI-Megatron-LM-240718/megatron/core/dist_checkpointing/__init__.py
--- a/PAI-Megatron-LM-240718/megatron/core/dist_checkpointing/core.py
+++ b/PAI-Megatron-LM-240718/megatron/core/dist_checkpointing/core.py
--- a/PAI-Megatron-LM-240718/megatron/core/dist_checkpointing/dict_utils.py
+++ b/PAI-Megatron-LM-240718/megatron/core/dist_checkpointing/dict_utils.py