Unverified Commit 2c05b8a5 authored by Funtowicz Morgan's avatar Funtowicz Morgan Committed by GitHub
Browse files

Remove tqdm logging when using pipelines. (#3833)

Introduce tqdm_enabled parameter on squad_convert_examples_to_features() default to True and set to False in QA pipelines.
parent c79b550d
......@@ -263,7 +263,15 @@ def squad_convert_example_to_features_init(tokenizer_for_convert):
def squad_convert_examples_to_features(
examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False, threads=1
examples,
tokenizer,
max_seq_length,
doc_stride,
max_query_length,
is_training,
return_dataset=False,
threads=1,
tqdm_enabled=True,
):
"""
Converts a list of examples into a list of features that can be directly given as input to a model.
......@@ -316,12 +324,15 @@ def squad_convert_examples_to_features(
p.imap(annotate_, examples, chunksize=32),
total=len(examples),
desc="convert squad examples to features",
disable=not tqdm_enabled,
)
)
new_features = []
unique_id = 1000000000
example_index = 0
for example_features in tqdm(features, total=len(features), desc="add example index and unique id"):
for example_features in tqdm(
features, total=len(features), desc="add example index and unique id", disable=not tqdm_enabled
):
if not example_features:
continue
for example_feature in example_features:
......
......@@ -958,6 +958,7 @@ class QuestionAnsweringPipeline(Pipeline):
kwargs["doc_stride"],
kwargs["max_question_len"],
False,
tqdm_enabled=False,
)
for example in examples
]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment