Unverified Commit d7906165 authored by Oren Amsalem's avatar Oren Amsalem Committed by GitHub
Browse files

add progress bar for convert_examples_to_features

It takes considerate amount of time (~10 min) to parse the examples to features, it is good to have a progress-bar to track this
parent d2e2577d
......@@ -23,6 +23,7 @@ import logging
import math
import collections
from io import open
from tqdm import tqdm
from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
......@@ -202,7 +203,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
# f = np.zeros((max_N, max_M), dtype=np.float32)
features = []
for (example_index, example) in enumerate(examples):
for (example_index, example) in enumerate(tqdm(examples)):
# if example_index % 100 == 0:
# logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment