Skip to content

GitLab

Unverified Commit f4ff44a6 authored by Cola's avatar Cola Committed by GitHub
Browse files

Fix batch_encode_plus (#3041)

parent b370cc7e
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 2 deletions
src/transformers/tokenization_utils.py
View file @ f4ff44a6
...@@ -1126,8 +1126,7 @@ class PreTrainedTokenizer(object): ...@@ -1126,8 +1126,7 @@ class PreTrainedTokenizer(object):
input_ids = [] input_ids = []
for ids_or_pair_ids in batch_text_or_text_pairs: for ids_or_pair_ids in batch_text_or_text_pairs:
if isinstance(ids_or_pair_ids, (list, tuple)): if isinstance(ids_or_pair_ids, (list, tuple)) and len(ids_or_pair_ids) == 2:
assert len(ids_or_pair_ids) == 2
ids, pair_ids = ids_or_pair_ids ids, pair_ids = ids_or_pair_ids
else: else:
ids, pair_ids = ids_or_pair_ids, None ids, pair_ids = ids_or_pair_ids, None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment