Create fairseq_cli_lib

Summary: This allows one to call fairseq_cli functions from within python without dispatching to bash.

Reviewed By: myleott

Differential Revision: D14404719

fbshipit-source-id: 044eb652045bb15fc40e72ecbaf6fb10df9f8c61

generate.py

@@ -11,7 +11,7 @@ Translate pre-processed data with a trained model.
 
 import torch
 
-from fairseq import bleu, options, progress_bar, tasks, tokenizer, utils
+from fairseq import bleu, options, progress_bar, tasks, utils
 from fairseq.meters import StopwatchMeter, TimeMeter
 from fairseq.utils import import_user_module
 
@@ -179,6 +179,7 @@ def main(args):
         num_sentences, gen_timer.n, gen_timer.sum, num_sentences / gen_timer.sum, 1. / gen_timer.avg))
     if has_target:
         print('| Generate {} with beam={}: {}'.format(args.gen_subset, args.beam, scorer.result_string()))
+    return scorer
 
 
 def cli_main():