第一次测试提交

bc5c7fa7 · wxj · 70fddd0f · bc5c7fa7 · bc5c7fa7 · bc5c7fa7
Commit bc5c7fa7 authored Jan 07, 2025 by wxj
20 changed files
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/blended_dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/blended_dataset.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/blended_megatron_dataset_builder.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/blended_megatron_dataset_builder.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/blended_megatron_dataset_config.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/blended_megatron_dataset_config.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/gpt_dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/gpt_dataset.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/helpers.cpp
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/helpers.cpp
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/indexed_dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/indexed_dataset.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/masked_dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/masked_dataset.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/megatron_dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/megatron_dataset.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/megatron_tokenizer.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/megatron_tokenizer.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/multimodal_dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/multimodal_dataset.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/readme.md
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/readme.md
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/__init__.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/__init__.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/__init__.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/__init__.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/bert_embedders.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/bert_embedders.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/config.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/config.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/gpt_chunk_datasets.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/gpt_chunk_datasets.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/tokenizers.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/config/tokenizers.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/db/__init__.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/db/__init__.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/db/build.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/db/build.py
--- a/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/db/dataset.py
+++ b/Megatron-LM-core_r0.7.0.beta/megatron/core/datasets/retro/db/dataset.py