Unverified Commit 7abc1d96 authored by Sam Shleifer's avatar Sam Shleifer Committed by GitHub
Browse files

no warn (#8329)

parent 52f44dd6
...@@ -269,7 +269,7 @@ class Seq2SeqDataCollator: ...@@ -269,7 +269,7 @@ class Seq2SeqDataCollator:
), f"pad_token_id is not defined for ({self.tokenizer.__class__.__name__}), it must be defined." ), f"pad_token_id is not defined for ({self.tokenizer.__class__.__name__}), it must be defined."
self.data_args = data_args self.data_args = data_args
self.tpu_num_cores = tpu_num_cores self.tpu_num_cores = tpu_num_cores
self.dataset_kwargs = {"add_prefix_space": isinstance(tokenizer, BartTokenizer)} self.dataset_kwargs = {"add_prefix_space": True} if isinstance(tokenizer, BartTokenizer) else {}
if data_args.src_lang is not None: if data_args.src_lang is not None:
self.dataset_kwargs["src_lang"] = data_args.src_lang self.dataset_kwargs["src_lang"] = data_args.src_lang
if data_args.tgt_lang is not None: if data_args.tgt_lang is not None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment