First add

d3dd8642 · Rayyyyy · d3dd8642 · d3dd8642 · d3dd8642 · d3dd8642
Commit d3dd8642 authored Jun 26, 2024 by Rayyyyy
20 changed files
--- a/tools/openwebtext/group_duplicate_url.py
+++ b/tools/openwebtext/group_duplicate_url.py
--- a/tools/openwebtext/merge_jsons.py
+++ b/tools/openwebtext/merge_jsons.py
--- a/tools/openwebtext/remove_group_duplicates.py
+++ b/tools/openwebtext/remove_group_duplicates.py
--- a/tools/preprocess_data.py
+++ b/tools/preprocess_data.py
--- a/tools/preprocess_data_nmt.py
+++ b/tools/preprocess_data_nmt.py
--- a/tools/preprocess_data_yuan.py
+++ b/tools/preprocess_data_yuan.py
--- a/tools/preprocess_mmdata.py
+++ b/tools/preprocess_mmdata.py
--- a/tools/retro/README.md
+++ b/tools/retro/README.md
--- a/tools/retro/cli/__init__.py
+++ b/tools/retro/cli/__init__.py
--- a/tools/retro/cli/__main__.py
+++ b/tools/retro/cli/__main__.py
--- a/tools/retro/cli/cli.py
+++ b/tools/retro/cli/cli.py
--- a/tools/retro/db/__init__.py
+++ b/tools/retro/db/__init__.py
--- a/tools/retro/db/build.py
+++ b/tools/retro/db/build.py
--- a/tools/retro/db/dataset.py
+++ b/tools/retro/db/dataset.py
--- a/tools/retro/db/utils.py
+++ b/tools/retro/db/utils.py
--- a/tools/retro/examples/preprocess_data.sh
+++ b/tools/retro/examples/preprocess_data.sh
--- a/tools/retro/examples/pretrain_model.sh
+++ b/tools/retro/examples/pretrain_model.sh
--- a/tools/retro/external_libs.py
+++ b/tools/retro/external_libs.py
--- a/tools/retro/index/__init__.py
+++ b/tools/retro/index/__init__.py
--- a/tools/retro/index/build.py
+++ b/tools/retro/index/build.py