Initial commit

0816dd4a · libo11 · 0816dd4a · 0816dd4a · 0816dd4a · 0816dd4a
Commit 0816dd4a authored Sep 29, 2024 by libo11
20 changed files
--- a/tools/bert_embedding/__init__.py
+++ b/tools/bert_embedding/__init__.py
--- a/tools/bert_embedding/dataset.py
+++ b/tools/bert_embedding/dataset.py
--- a/tools/bert_embedding/embed.py
+++ b/tools/bert_embedding/embed.py
--- a/tools/bert_embedding/external_libs.py
+++ b/tools/bert_embedding/external_libs.py
--- a/tools/bert_embedding/huggingface.py
+++ b/tools/bert_embedding/huggingface.py
--- a/tools/checkpoint/convert.py
+++ b/tools/checkpoint/convert.py
--- a/tools/checkpoint/hybrid_conversion.py
+++ b/tools/checkpoint/hybrid_conversion.py
--- a/tools/checkpoint/loader_llama_mistral.py
+++ b/tools/checkpoint/loader_llama_mistral.py
--- a/tools/checkpoint/loader_mcore.py
+++ b/tools/checkpoint/loader_mcore.py
--- a/tools/checkpoint/loader_megatron.py
+++ b/tools/checkpoint/loader_megatron.py
--- a/tools/checkpoint/saver_mcore.py
+++ b/tools/checkpoint/saver_mcore.py
--- a/tools/checkpoint/saver_megatron.py
+++ b/tools/checkpoint/saver_megatron.py
--- a/tools/checkpoint/setter.py
+++ b/tools/checkpoint/setter.py
--- a/tools/checkpoint/utils.py
+++ b/tools/checkpoint/utils.py
--- a/tools/linter.py
+++ b/tools/linter.py
--- a/tools/merge_datasets.py
+++ b/tools/merge_datasets.py
--- a/tools/openwebtext/README.md
+++ b/tools/openwebtext/README.md
--- a/tools/openwebtext/add_id.py
+++ b/tools/openwebtext/add_id.py
--- a/tools/openwebtext/blacklist_urls.py
+++ b/tools/openwebtext/blacklist_urls.py
--- a/tools/openwebtext/cleanup_dataset.py
+++ b/tools/openwebtext/cleanup_dataset.py