Unverified Commit d7906165 authored by Oren Amsalem's avatar Oren Amsalem Committed by GitHub
Browse files

add progress bar for convert_examples_to_features

It takes considerate amount of time (~10 min) to parse the examples to features, it is good to have a progress-bar to track this
parent d2e2577d
...@@ -23,6 +23,7 @@ import logging ...@@ -23,6 +23,7 @@ import logging
import math import math
import collections import collections
from io import open from io import open
from tqdm import tqdm
from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
...@@ -202,7 +203,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length, ...@@ -202,7 +203,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
# f = np.zeros((max_N, max_M), dtype=np.float32) # f = np.zeros((max_N, max_M), dtype=np.float32)
features = [] features = []
for (example_index, example) in enumerate(examples): for (example_index, example) in enumerate(tqdm(examples)):
# if example_index % 100 == 0: # if example_index % 100 == 0:
# logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg) # logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment