Remove 50k limits bug

8e093e59 · peterandluc · Julien Chaumond · 6af5a54c · 8e093e59
Commit 8e093e59 authored Apr 23, 2020 by peterandluc Committed by Julien Chaumond Apr 23, 2020
Show whitespace changes
Inline Side-by-side

Showing with 0 additions and 1 deletion

src/transformers/data/datasets/language_modeling.py src/transformers/data/datasets/language_modeling.py +0 -1

No files found.
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -90,7 +90,6 @@ class LineByLineTextDataset(Dataset):
        with open(file_path, encoding="utf-8") as f:
            lines = [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]

-        lines = lines[:50_000]
        batch_encoding = tokenizer.batch_encode_plus(lines, add_special_tokens=True, max_length=block_size)
        self.examples = batch_encoding["input_ids"]