"git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "9535bf1977b33ebaf34363cf4d0b305157194015"
Unverified Commit 61e068e5 authored by Dudu Lasry's avatar Dudu Lasry Committed by GitHub
Browse files

Support turning off the model uploading in ClearML (#20969)



* Add support for turning off the model uploading in ClearML

* Add documentation for the CLEARML_LOG_MODEL environment variable

* Adjust new doc addition to the new style
Co-authored-by: default avatarSylvain Gugger <35901082+sgugger@users.noreply.github.com>
Co-authored-by: default avatarDudu Lasry <dudu.lasry@viz.ai>
Co-authored-by: default avatarSylvain Gugger <35901082+sgugger@users.noreply.github.com>
parent ff8dcb5e
...@@ -1313,6 +1313,8 @@ class ClearMLCallback(TrainerCallback): ...@@ -1313,6 +1313,8 @@ class ClearMLCallback(TrainerCallback):
ClearML project name. ClearML project name.
- **CLEARML_TASK** (`str`, *optional*, defaults to `Trainer`): - **CLEARML_TASK** (`str`, *optional*, defaults to `Trainer`):
ClearML task name. ClearML task name.
- **CLEARML_LOG_MODEL** (`bool`, *optional*, defaults to `False`):
Whether to log models as artifacts during training.
""" """
def __init__(self): def __init__(self):
...@@ -1326,6 +1328,8 @@ class ClearMLCallback(TrainerCallback): ...@@ -1326,6 +1328,8 @@ class ClearMLCallback(TrainerCallback):
self._initialized = False self._initialized = False
self._clearml_task = None self._clearml_task = None
self._log_model = os.getenv("CLEARML_LOG_MODEL", "FALSE").upper() in ENV_VARS_TRUE_VALUES.union({"TRUE"})
def setup(self, args, state, model, tokenizer, **kwargs): def setup(self, args, state, model, tokenizer, **kwargs):
if self._clearml is None: if self._clearml is None:
return return
...@@ -1403,7 +1407,7 @@ class ClearMLCallback(TrainerCallback): ...@@ -1403,7 +1407,7 @@ class ClearMLCallback(TrainerCallback):
) )
def on_save(self, args, state, control, **kwargs): def on_save(self, args, state, control, **kwargs):
if self._clearml_task and state.is_world_process_zero: if self._log_model and self._clearml_task and state.is_world_process_zero:
ckpt_dir = f"checkpoint-{state.global_step}" ckpt_dir = f"checkpoint-{state.global_step}"
artifact_path = os.path.join(args.output_dir, ckpt_dir) artifact_path = os.path.join(args.output_dir, ckpt_dir)
logger.info(f"Logging checkpoint artifacts in {ckpt_dir}. This may take time.") logger.info(f"Logging checkpoint artifacts in {ckpt_dir}. This may take time.")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment