Merge branch 'main' of https://github.com/NVIDIA/Megatron-LM

0024a5c6 · zhuwenwen · b004456b · 3db2063b · 0024a5c6 · 0024a5c6
Commit 0024a5c6 authored Apr 26, 2023 by zhuwenwen
20 changed files
--- a/tools/openwebtext/remove_group_duplicates.py
+++ b/tools/openwebtext/remove_group_duplicates.py
--- a/tools/preprocess_data.py
+++ b/tools/preprocess_data.py
--- a/tools/preprocess_data_nmt.py
+++ b/tools/preprocess_data_nmt.py
--- a/tools/preprocess_data_partitions.py
+++ b/tools/preprocess_data_partitions.py
--- a/tools/retro/README.md
+++ b/tools/retro/README.md
--- a/tools/retro/cli/__init__.py
+++ b/tools/retro/cli/__init__.py
--- a/tools/retro/cli/__main__.py
+++ b/tools/retro/cli/__main__.py
--- a/tools/retro/db/__init__.py
+++ b/tools/retro/db/__init__.py
--- a/tools/retro/db/build.py
+++ b/tools/retro/db/build.py
--- a/tools/retro/db/dataset.py
+++ b/tools/retro/db/dataset.py
--- a/tools/retro/db/utils.py
+++ b/tools/retro/db/utils.py
--- a/tools/retro/examples/get_dataset_configs.sh
+++ b/tools/retro/examples/get_dataset_configs.sh
--- a/tools/retro/examples/get_preprocess_cmd.sh
+++ b/tools/retro/examples/get_preprocess_cmd.sh
--- a/tools/retro/examples/preprocess_data.sh
+++ b/tools/retro/examples/preprocess_data.sh
--- a/tools/retro/examples/pretrain_model.sh
+++ b/tools/retro/examples/pretrain_model.sh
--- a/tools/retro/external_libs.py
+++ b/tools/retro/external_libs.py
--- a/tools/retro/index/__init__.py
+++ b/tools/retro/index/__init__.py
--- a/tools/retro/index/build.py
+++ b/tools/retro/index/build.py
--- a/tools/retro/index/factory.py
+++ b/tools/retro/index/factory.py
--- a/tools/retro/index/index.py
+++ b/tools/retro/index/index.py