get_wkt2.sh

#!/bin/bash

"""
This file is adapted from
https://github.com/salesforce/awd-lstm-lm/blob/master/getdata.sh
Copyright by the AWD LSTM authors.
"""
DATA_DIR=$1
echo "- Downloading WikiText-2"

wget --continue -P $DATA_DIR https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip
unzip -q $DATA_DIR/wikitext-2-v1.zip -d $DATA_DIR
cd $DATA_DIR/wikitext-2
mv wiki.train.tokens train.txt
sed -i -e "s/<unk>/[UNK]/g" train.txt
mv wiki.valid.tokens valid.txt
sed -i -e "s/<unk>/[UNK]/g" valid.txt
mv wiki.test.tokens test.txt
sed -i -e "s/<unk>/[UNK]/g" test.txt
cd ..
rm wikitext-2-v1.zip

echo "- WikiText-2 saved at $DATA_DIR/wikitext-2"