"tests/vscode:/vscode.git/clone" did not exist on "2de5cb12be7b7b3d5cf1f51880897a40879cb805"
Unverified Commit 0b5024ce authored by Younes Belkada's avatar Younes Belkada Committed by GitHub
Browse files

[`Trainer`] Refactor trainer + bnb logic (#26248)

* refactor trainer + bnb logic

* remove logger.info

* oops
parent f94c9b3d
...@@ -402,15 +402,19 @@ class Trainer: ...@@ -402,15 +402,19 @@ class Trainer:
" to `True` to avoid any unexpected behavior such as device placement mismatching." " to `True` to avoid any unexpected behavior such as device placement mismatching."
) )
_is_peft_model = is_peft_available() and isinstance(model, PeftModel)
_is_quantized_and_base_model = getattr(model, "is_quantized", False) and not getattr(
model, "_hf_peft_config_loaded", False
)
# At this stage the model is already loaded # At this stage the model is already loaded
if getattr(model, "is_quantized", False) and not getattr(model, "_hf_peft_config_loaded", False): if _is_quantized_and_base_model and not _is_peft_model:
if getattr(model, "_is_quantized_training_enabled", False): raise ValueError(
logger.info( "You cannot perform fine-tuning on purely quantized models. Please attach trainable adapters on top of"
"The model is quantized. To train this model you need to add additional modules" " the quantized model to correctly perform fine-tuning. Please see: https://huggingface.co/docs/transformers/peft"
" inside the model such as adapters using `peft` library and freeze the model weights. Please" " for more details"
" check the examples in https://github.com/huggingface/peft for more details."
) )
else: elif _is_quantized_and_base_model and not getattr(model, "_is_quantized_training_enabled", False):
raise ValueError( raise ValueError(
"The model you want to train is loaded in 8-bit precision. if you want to fine-tune an 8-bit" "The model you want to train is loaded in 8-bit precision. if you want to fine-tune an 8-bit"
" model, please make sure that you have installed `bitsandbytes>=0.37.0`. " " model, please make sure that you have installed `bitsandbytes>=0.37.0`. "
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment