Removed dataset limit

5c18825a · LysandreJik · 3e3e1454 · 5c18825a
Commit 5c18825a authored Aug 06, 2019 by LysandreJik
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

examples/utils_lm.py examples/utils_lm.py +1 -1

No files found.
--- a/examples/utils_lm.py
+++ b/examples/utils_lm.py
@@ -14,7 +14,7 @@ class WikiTextDataset(Dataset):
 		with open(os.path.join(directory, f"wiki.{file}.raw"), encoding="utf-8") as f:
 			text = f.read()
-			spans = list(filter(lambda item: len(item) > 120, text.split("\n")[:20]))
+			spans = list(filter(lambda item: len(item) > 120, text.split("\n")))
 			for span in spans:
 				span = tokenizer.encode(span)