Missed some changes from next-best-lm branch.

megatron/global_vars.py

@@ -89,7 +89,7 @@ def set_global_variables(args):
     set_args(args)
 
     _build_num_microbatches_calculator(args)
-    if args.vocab_file:
+    if args.vocab_file or args.tokenizer_model:
         _ = _build_tokenizer(args)
     _set_tensorboard_writer(args)
     _set_adlr_autoresume(args)

tools/preprocess_data.py

@@ -104,6 +104,8 @@ def get_args():
                        help='Append an <eod> token to the end of a document.')
     group.add_argument('--lang', type=str, default='english',
                        help='Language to use for NLTK-powered sentence splitting.')
+    group.add_argument('--tokenizer-model', type=str, default=None,
+                       help='sentencepeice tokenizer model.')
 
 
     group = parser.add_argument_group(title='output data')

tools/preprocess_data_partitions.py

@@ -326,6 +326,9 @@ def main():
         for p in processes:
             p.join()
 
+        if args.partitions == 1:
+            return
+
 
     # encode partition files in parallel
     processes = []