Initial commit

0816dd4a · libo11 · 0816dd4a · 0816dd4a · 0816dd4a · 0816dd4a
Commit 0816dd4a authored Sep 29, 2024 by libo11
20 changed files
--- a/tools/openwebtext/cleanup_fix_dataset.py
+++ b/tools/openwebtext/cleanup_fix_dataset.py
--- a/tools/openwebtext/filter_ngrams.py
+++ b/tools/openwebtext/filter_ngrams.py
--- a/tools/openwebtext/find_duplicates.py
+++ b/tools/openwebtext/find_duplicates.py
--- a/tools/openwebtext/group_duplicate_url.py
+++ b/tools/openwebtext/group_duplicate_url.py
--- a/tools/openwebtext/merge_jsons.py
+++ b/tools/openwebtext/merge_jsons.py
--- a/tools/openwebtext/remove_group_duplicates.py
+++ b/tools/openwebtext/remove_group_duplicates.py
--- a/tools/preprocess_data.py
+++ b/tools/preprocess_data.py
--- a/tools/preprocess_data_nmt.py
+++ b/tools/preprocess_data_nmt.py
--- a/tools/preprocess_mmdata.py
+++ b/tools/preprocess_mmdata.py
--- a/tools/report_theoretical_memory.py
+++ b/tools/report_theoretical_memory.py
--- a/tools/retro/README.md
+++ b/tools/retro/README.md
--- a/tools/retro/build_db.md
+++ b/tools/retro/build_db.md
--- a/tools/retro/cli/__init__.py
+++ b/tools/retro/cli/__init__.py
--- a/tools/retro/cli/__main__.py
+++ b/tools/retro/cli/__main__.py
--- a/tools/retro/cli/cli.py
+++ b/tools/retro/cli/cli.py
--- a/tools/retro/config_utils.py
+++ b/tools/retro/config_utils.py
--- a/tools/retro/docker/Dockerfile
+++ b/tools/retro/docker/Dockerfile
--- a/tools/retro/preprocess_data.py
+++ b/tools/retro/preprocess_data.py
--- a/tools/retro/sft/README.md
+++ b/tools/retro/sft/README.md
--- a/tools/retro/sft/dataset_conv.py
+++ b/tools/retro/sft/dataset_conv.py