Skip to content

GitLab

Unverified Commit edf9ac11 authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Should return overflowing information for the log (#4385)

parent b908f2e9
Hide whitespace changes
Inline Side-by-side
Showing with 6 additions and 1 deletion
examples/multiple-choice/utils_multiple_choice.py
View file @ edf9ac11
...@@ -535,7 +535,12 @@ def convert_examples_to_features( ...@@ -535,7 +535,12 @@ def convert_examples_to_features(
text_b = example.question + " " + ending text_b = example.question + " " + ending
inputs = tokenizer.encode_plus( inputs = tokenizer.encode_plus(
text_a, text_b, add_special_tokens=True, max_length=max_length, pad_to_max_length=True, text_a,
text_b,
add_special_tokens=True,
max_length=max_length,
pad_to_max_length=True,
return_overflowing_tokens=True,
) )
if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0: if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0:
logger.info( logger.info(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment