Update GPT-2 BPE

Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/749

Differential Revision: D16410984

Pulled By: myleott

fbshipit-source-id: 7698df46b8a179afccb287990f9705358690454a

fairseq/data/encoders/gpt2_bpe.py

@@ -9,6 +9,10 @@ from fairseq import file_utils
 from fairseq.data.encoders import register_bpe
 
 
+DEFAULT_ENCODER_JSON = 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json'
+DEFAULT_VOCAB_BPE = 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe'
+
+
 @register_bpe('gpt2')
 class GPT2BPE(object):
 
@@ -16,16 +20,20 @@ class GPT2BPE(object):
     def add_args(parser):
         # fmt: off
         parser.add_argument('--gpt2-encoder-json', type=str,
-                            default='https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json',
+                            default=DEFAULT_ENCODER_JSON,
                             help='path to encoder.json')
         parser.add_argument('--gpt2-vocab-bpe', type=str,
-                            default='https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe',
+                            default=DEFAULT_VOCAB_BPE,
                             help='path to vocab.bpe')
         # fmt: on
 
     def __init__(self, args):
-        encoder_json = file_utils.cached_path(args.gpt2_encoder_json)
-        vocab_bpe = file_utils.cached_path(args.gpt2_vocab_bpe)
+        encoder_json = file_utils.cached_path(
+            getattr(args, 'gpt2_encoder_json', DEFAULT_ENCODER_JSON)
+        )
+        vocab_bpe = file_utils.cached_path(
+            getattr(args, 'gpt2_vocab_bpe', DEFAULT_VOCAB_BPE)
+        )
         self.bpe = get_encoder(encoder_json, vocab_bpe)
 
     def encode(self, x: str) -> str: