deprecated data_utils

8600642e · Mohammad · deffcb6a · 8600642e · 8600642e · 8600642e
Commit 8600642e authored Mar 31, 2020 by Mohammad
10 changed files
--- a/megatron/data_utils/__init__.py
+++ b/megatron/data_utils/__init__.py
--- a/megatron/data_utils/corpora.py
+++ b/megatron/data_utils/corpora.py
--- a/megatron/data_utils/datasets.py
+++ b/megatron/data_utils/datasets.py
--- a/megatron/data_utils/file_utils.py
+++ b/megatron/data_utils/file_utils.py
--- a/megatron/data_utils/lazy_loader.py
+++ b/megatron/data_utils/lazy_loader.py
--- a/megatron/data_utils/samplers.py
+++ b/megatron/data_utils/samplers.py
--- a/megatron/data_utils/tf_dl.py
+++ b/megatron/data_utils/tf_dl.py
--- a/megatron/data_utils/tokenization.py
+++ b/megatron/data_utils/tokenization.py
--- a/megatron/data_utils/tokenization_gpt2.py
+++ b/megatron/data_utils/tokenization_gpt2.py
--- a/megatron/data_utils/wordpiece.py
+++ b/megatron/data_utils/wordpiece.py