Skip to content

GitLab

Unverified Commit cec011bb authored by anj-s's avatar anj-s Committed by GitHub
Browse files

[chore]Migrate away from legacy torchtext iterators (#713) 

* migrate away from legacy iterators

* fix lint error
parent cbeda830
Hide whitespace changes
Inline Side-by-side
Showing with 8 additions and 1 deletion
benchmarks/datasets/wikitext2_data.py
View file @ cec011bb
...@@ -3,14 +3,21 @@ ...@@ -3,14 +3,21 @@
# This source code is licensed under the BSD license found in the # This source code is licensed under the BSD license found in the
# LICENSE file in the root directory of this source tree. # LICENSE file in the root directory of this source tree.
from distutils.version import LooseVersion
import io import io
import operator
import tempfile import tempfile
import torch import torch
from torch.utils.data import DataLoader from torch.utils.data import DataLoader
import torchtext
from torchtext.data.utils import get_tokenizer from torchtext.data.utils import get_tokenizer
from torchtext.utils import download_from_url, extract_archive from torchtext.utils import download_from_url, extract_archive
from torchtext.vocab import build_vocab_from_iterator
if operator.ge(torchtext.__version__, LooseVersion("0.10.0")):
from torchtext.legacy.vocab import build_vocab_from_iterator
else:
from torchtext.vocab import build_vocab_from_iterator
def _batchify(data, batch_size): def _batchify(data, batch_size):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment