added threads

a29890c3 · Mostofa Patwary · 90c9e3f2 · a29890c3
Commit a29890c3 authored Feb 26, 2021 by Mostofa Patwary
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

tools/openwebtext/filter_ngrams.py tools/openwebtext/filter_ngrams.py +1 -1

No files found.
--- a/tools/openwebtext/filter_ngrams.py
+++ b/tools/openwebtext/filter_ngrams.py
@@ -284,7 +284,7 @@ if __name__ == '__main__':
    dedup_key = args.dedup_dataset[1]

    # Setup multi-processing.
-    num_workers = 1 #40
+    num_workers = 40
    fin = open(dedup_file, 'r', encoding='utf-8')
    pool = multiprocessing.Pool(num_workers)
    free_ngram_x=partial(free_ngram, args=args, key=dedup_key, ngrams=ngrams, \