stitch preprocessing pipeline

Summary:
1. add call to binarization to complete preprocessing pipeline
2. add ability to specify task to select the dictionary, and add a bert task
3. Get rid of function calls that are no longer needed after moving functions from fairseq here

Reviewed By: jingfeidu

Differential Revision: D13977842

fbshipit-source-id: ec9bbb4e98e62e12c20ba68bb52b8bcc94aee91d

preprocess.py

@@ -232,7 +232,7 @@ def main(args):
                 print("{} {}".format(src_dict[k], tgt_dict[v]), file=f)
 
 
-def binarize(args, filename, dict, output_prefix, lang, offset, end):
+def binarize(args, filename, dict, output_prefix, lang, offset, end, append_eos=True):
     ds = indexed_dataset.IndexedDatasetBuilder(
         dataset_dest_file(args, output_prefix, lang, "bin")
     )
@@ -240,7 +240,14 @@ def binarize(args, filename, dict, output_prefix, lang, offset, end):
     def consumer(tensor):
         ds.add_item(tensor)
 
-    res = Tokenizer.binarize(filename, dict, consumer, offset=offset, end=end)
+    res = Tokenizer.binarize(
+        filename,
+        dict,
+        consumer,
+        offset=offset,
+        end=end,
+        append_eos=append_eos
+    )
     ds.finalize(dataset_dest_file(args, output_prefix, lang, "idx"))
     return res