Skip to content

GitLab

Commit 2e4db64c authored by thomwolf's avatar thomwolf
Browse files

add do_lower_case tokenizer loading optino in run_squad and ine_tuning examples

parent c9fd3505
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
examples/run_lm_finetuning.py
View file @ 2e4db64c
...@@ -515,7 +515,7 @@ def main(): ...@@ -515,7 +515,7 @@ def main():
raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
os.makedirs(args.output_dir, exist_ok=True) os.makedirs(args.output_dir, exist_ok=True)
tokenizer = BertTokenizer.from_pretrained(args.bert_model) tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
#train_examples = None #train_examples = None
num_train_steps = None num_train_steps = None
......
examples/run_squad.py
View file @ 2e4db64c
...@@ -786,7 +786,7 @@ def main(): ...@@ -786,7 +786,7 @@ def main():
raise ValueError("Output directory () already exists and is not empty.") raise ValueError("Output directory () already exists and is not empty.")
os.makedirs(args.output_dir, exist_ok=True) os.makedirs(args.output_dir, exist_ok=True)
tokenizer = BertTokenizer.from_pretrained(args.bert_model) tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
train_examples = None train_examples = None
num_train_steps = None num_train_steps = None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment