Unverified Commit 2c05b8a5 authored by Funtowicz Morgan's avatar Funtowicz Morgan Committed by GitHub
Browse files

Remove tqdm logging when using pipelines. (#3833)

Introduce tqdm_enabled parameter on squad_convert_examples_to_features() default to True and set to False in QA pipelines.
parent c79b550d
...@@ -263,7 +263,15 @@ def squad_convert_example_to_features_init(tokenizer_for_convert): ...@@ -263,7 +263,15 @@ def squad_convert_example_to_features_init(tokenizer_for_convert):
def squad_convert_examples_to_features( def squad_convert_examples_to_features(
examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False, threads=1 examples,
tokenizer,
max_seq_length,
doc_stride,
max_query_length,
is_training,
return_dataset=False,
threads=1,
tqdm_enabled=True,
): ):
""" """
Converts a list of examples into a list of features that can be directly given as input to a model. Converts a list of examples into a list of features that can be directly given as input to a model.
...@@ -316,12 +324,15 @@ def squad_convert_examples_to_features( ...@@ -316,12 +324,15 @@ def squad_convert_examples_to_features(
p.imap(annotate_, examples, chunksize=32), p.imap(annotate_, examples, chunksize=32),
total=len(examples), total=len(examples),
desc="convert squad examples to features", desc="convert squad examples to features",
disable=not tqdm_enabled,
) )
) )
new_features = [] new_features = []
unique_id = 1000000000 unique_id = 1000000000
example_index = 0 example_index = 0
for example_features in tqdm(features, total=len(features), desc="add example index and unique id"): for example_features in tqdm(
features, total=len(features), desc="add example index and unique id", disable=not tqdm_enabled
):
if not example_features: if not example_features:
continue continue
for example_feature in example_features: for example_feature in example_features:
......
...@@ -958,6 +958,7 @@ class QuestionAnsweringPipeline(Pipeline): ...@@ -958,6 +958,7 @@ class QuestionAnsweringPipeline(Pipeline):
kwargs["doc_stride"], kwargs["doc_stride"],
kwargs["max_question_len"], kwargs["max_question_len"],
False, False,
tqdm_enabled=False,
) )
for example in examples for example in examples
] ]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment