Skip to content

GitLab

Unverified Commit aca6288f authored by Bhadresh Savani's avatar Bhadresh Savani Committed by GitHub
Browse files

updated logging and saving metrics (#10436) 

* updated logging and saving metrics

* space removal
parent f52a1589
Hide whitespace changes
Inline Side-by-side
Showing with 71 additions and 191 deletions
examples/language-modeling/run_clm.py
View file @ aca6288f
...@@ -375,16 +375,11 @@ def main(): ...@@ -375,16 +375,11 @@ def main():
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") metrics = train_result.metrics
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model trainer.log_metrics("train", metrics)
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json")) trainer.save_metrics("train", metrics)
trainer.save_state()
# Evaluation # Evaluation
results = {} results = {}
...@@ -396,13 +391,8 @@ def main(): ...@@ -396,13 +391,8 @@ def main():
perplexity = math.exp(eval_output["eval_loss"]) perplexity = math.exp(eval_output["eval_loss"])
results["perplexity"] = perplexity results["perplexity"] = perplexity
output_eval_file = os.path.join(training_args.output_dir, "eval_results_clm.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
return results return results
......
examples/language-modeling/run_mlm.py
View file @ aca6288f
...@@ -411,17 +411,11 @@ def main(): ...@@ -411,17 +411,11 @@ def main():
checkpoint = None checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") trainer.log_metrics("train", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_state()
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
results = {} results = {}
...@@ -433,13 +427,8 @@ def main(): ...@@ -433,13 +427,8 @@ def main():
perplexity = math.exp(eval_output["eval_loss"]) perplexity = math.exp(eval_output["eval_loss"])
results["perplexity"] = perplexity results["perplexity"] = perplexity
output_eval_file = os.path.join(training_args.output_dir, "eval_results_mlm.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
return results return results
......
examples/language-modeling/run_plm.py
View file @ aca6288f
...@@ -392,17 +392,11 @@ def main(): ...@@ -392,17 +392,11 @@ def main():
checkpoint = None checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") trainer.log_metrics("train", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_state()
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
results = {} results = {}
...@@ -414,13 +408,8 @@ def main(): ...@@ -414,13 +408,8 @@ def main():
perplexity = math.exp(eval_output["eval_loss"]) perplexity = math.exp(eval_output["eval_loss"])
results["perplexity"] = perplexity results["perplexity"] = perplexity
output_eval_file = os.path.join(training_args.output_dir, "eval_results_plm.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
return results return results
......
examples/multiple-choice/run_swag.py
View file @ aca6288f
...@@ -227,6 +227,8 @@ def main(): ...@@ -227,6 +227,8 @@ def main():
# Set the verbosity to info of the Transformers logger (on main process only): # Set the verbosity to info of the Transformers logger (on main process only):
if is_main_process(training_args.local_rank): if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info() transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logger.info("Training/evaluation parameters %s", training_args) logger.info("Training/evaluation parameters %s", training_args)
# Set seed before initializing model. # Set seed before initializing model.
...@@ -367,17 +369,11 @@ def main(): ...@@ -367,17 +369,11 @@ def main():
checkpoint = None checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") trainer.log_metrics("train", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_state()
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
results = {} results = {}
...@@ -386,13 +382,8 @@ def main(): ...@@ -386,13 +382,8 @@ def main():
results = trainer.evaluate() results = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results_swag.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
return results return results
......
examples/multiple-choice/run_tf_multiple_choice.py
View file @ aca6288f
...@@ -206,14 +206,10 @@ def main(): ...@@ -206,14 +206,10 @@ def main():
result = trainer.evaluate() result = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt") trainer.log_metrics("eval", results)
with open(output_eval_file, "w") as writer: trainer.save_metrics("eval", results)
logger.info("***** Eval results *****")
for key, value in result.items(): results.update(result)
logger.info(" %s = %s", key, value)
writer.write("%s = %s\n" % (key, value))
results.update(result)
return results return results
......
examples/question-answering/run_qa.py
View file @ aca6288f
...@@ -201,6 +201,8 @@ def main(): ...@@ -201,6 +201,8 @@ def main():
# Set the verbosity to info of the Transformers logger (on main process only): # Set the verbosity to info of the Transformers logger (on main process only):
if is_main_process(training_args.local_rank): if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info() transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logger.info("Training/evaluation parameters %s", training_args) logger.info("Training/evaluation parameters %s", training_args)
# Set seed before initializing model. # Set seed before initializing model.
...@@ -479,16 +481,10 @@ def main(): ...@@ -479,16 +481,10 @@ def main():
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") metrics = train_result.metrics
if trainer.is_world_process_zero(): trainer.log_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_metrics("train", metrics)
logger.info("***** Train results *****") trainer.save_state()
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
results = {} results = {}
...@@ -496,13 +492,8 @@ def main(): ...@@ -496,13 +492,8 @@ def main():
logger.info("*** Evaluate ***") logger.info("*** Evaluate ***")
results = trainer.evaluate() results = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
return results return results
......
examples/question-answering/run_qa_beam_search.py
View file @ aca6288f
...@@ -200,6 +200,8 @@ def main(): ...@@ -200,6 +200,8 @@ def main():
# Set the verbosity to info of the Transformers logger (on main process only): # Set the verbosity to info of the Transformers logger (on main process only):
if is_main_process(training_args.local_rank): if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info() transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logger.info("Training/evaluation parameters %s", training_args) logger.info("Training/evaluation parameters %s", training_args)
# Set seed before initializing model. # Set seed before initializing model.
...@@ -518,16 +520,10 @@ def main(): ...@@ -518,16 +520,10 @@ def main():
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") metrics = train_result.metrics
if trainer.is_world_process_zero(): trainer.log_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_metrics("train", metrics)
logger.info("***** Train results *****") trainer.save_state()
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
results = {} results = {}
...@@ -535,13 +531,8 @@ def main(): ...@@ -535,13 +531,8 @@ def main():
logger.info("*** Evaluate ***") logger.info("*** Evaluate ***")
results = trainer.evaluate() results = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
return results return results
......
examples/text-classification/run_glue.py
View file @ aca6288f
...@@ -417,16 +417,9 @@ def main(): ...@@ -417,16 +417,9 @@ def main():
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") trainer.log_metrics("train", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_state()
logger.info("***** Train results *****")
for key, value in sorted(metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
eval_results = {} eval_results = {}
...@@ -443,13 +436,8 @@ def main(): ...@@ -443,13 +436,8 @@ def main():
for eval_dataset, task in zip(eval_datasets, tasks): for eval_dataset, task in zip(eval_datasets, tasks):
eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_result = trainer.evaluate(eval_dataset=eval_dataset)
output_eval_file = os.path.join(training_args.output_dir, f"eval_results_{task}.txt") trainer.log_metrics("eval", eval_result)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", eval_result)
with open(output_eval_file, "w") as writer:
logger.info(f"***** Eval results {task} *****")
for key, value in sorted(eval_result.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
eval_results.update(eval_result) eval_results.update(eval_result)
......
examples/text-classification/run_tf_glue.py
View file @ aca6288f
...@@ -247,18 +247,10 @@ def main(): ...@@ -247,18 +247,10 @@ def main():
results = {} results = {}
if training_args.do_eval: if training_args.do_eval:
logger.info("*** Evaluate ***") logger.info("*** Evaluate ***")
result = trainer.evaluate() result = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt") trainer.log_metrics("eval", result)
trainer.save_metrics("eval", result)
with open(output_eval_file, "w") as writer: results.update(result)
logger.info("***** Eval results *****")
for key, value in result.items():
logger.info(" %s = %s", key, value)
writer.write("%s = %s\n" % (key, value))
results.update(result)
return results return results
......
examples/text-classification/run_tf_text_classification.py
View file @ aca6288f
...@@ -293,18 +293,10 @@ def main(): ...@@ -293,18 +293,10 @@ def main():
results = {} results = {}
if training_args.do_eval: if training_args.do_eval:
logger.info("*** Evaluate ***") logger.info("*** Evaluate ***")
result = trainer.evaluate() result = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt") trainer.log_metrics("eval", result)
trainer.save_metrics("eval", result)
with open(output_eval_file, "w") as writer: results.update(result)
logger.info("***** Eval results *****")
for key, value in result.items():
logger.info(" %s = %s", key, value)
writer.write("%s = %s\n" % (key, value))
results.update(result)
return results return results
......
examples/text-classification/run_xnli.py
View file @ aca6288f
...@@ -291,33 +291,19 @@ def main(): ...@@ -291,33 +291,19 @@ def main():
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") trainer.log_metrics("train", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_state()
logger.info("***** Train results *****")
for key, value in sorted(metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
eval_results = {} eval_results = {}
if training_args.do_eval: if training_args.do_eval:
logger.info("*** Evaluate ***") logger.info("*** Evaluate ***")
eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_result = trainer.evaluate(eval_dataset=eval_dataset)
output_eval_file = os.path.join(training_args.output_dir, "eval_results_xnli.txt") trainer.log_metrics("eval", eval_result)
trainer.save_metrics("eval", eval_result)
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results xnli *****")
for key, value in sorted(eval_result.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
eval_results.update(eval_result) eval_results.update(eval_result)
return eval_results return eval_results
......
examples/token-classification/run_ner.py
View file @ aca6288f
...@@ -387,18 +387,12 @@ def main(): ...@@ -387,18 +387,12 @@ def main():
else: else:
checkpoint = None checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint) train_result = trainer.train(resume_from_checkpoint=checkpoint)
metrics = train_result.metrics
trainer.save_model() # Saves the tokenizer too for easy upload trainer.save_model() # Saves the tokenizer too for easy upload
output_train_file = os.path.join(training_args.output_dir, "train_results.txt") trainer.log_metrics("train", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("train", metrics)
with open(output_train_file, "w") as writer: trainer.save_state()
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
# Evaluation # Evaluation
results = {} results = {}
...@@ -407,13 +401,8 @@ def main(): ...@@ -407,13 +401,8 @@ def main():
results = trainer.evaluate() results = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results_ner.txt") trainer.log_metrics("eval", results)
if trainer.is_world_process_zero(): trainer.save_metrics("eval", results)
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in results.items():
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Predict # Predict
if training_args.do_predict: if training_args.do_predict:
...@@ -429,12 +418,8 @@ def main(): ...@@ -429,12 +418,8 @@ def main():
for prediction, label in zip(predictions, labels) for prediction, label in zip(predictions, labels)
] ]
output_test_results_file = os.path.join(training_args.output_dir, "test_results.txt") trainer.log_metrics("test", metrics)
if trainer.is_world_process_zero(): trainer.save_metrics("test", metrics)
with open(output_test_results_file, "w") as writer:
for key, value in sorted(metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
# Save predictions # Save predictions
output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt") output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment