Skip to content

GitLab

".github/vscode:/vscode.git/clone" did not exist on "44ac25eae2cbbdc1cbcca423777107a5ca90a8f4"
Unverified Commit be0c5180 authored by Michael Goin's avatar Michael Goin Committed by GitHub
Browse files

[Bugfix] Add logs for all model dtype casting (#4717)

parent cea64430
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 0 deletions
vllm/config.py
View file @ be0c5180
...@@ -1063,6 +1063,7 @@ def _get_and_verify_dtype( ...@@ -1063,6 +1063,7 @@ def _get_and_verify_dtype(
if config_dtype == torch.float32: if config_dtype == torch.float32:
# Following the common practice, we use float16 for float32 # Following the common practice, we use float16 for float32
# models. # models.
logger.info("Casting torch.float32 to torch.float16.")
torch_dtype = torch.float16 torch_dtype = torch.float16
else: else:
torch_dtype = config_dtype torch_dtype = config_dtype
...@@ -1087,9 +1088,11 @@ def _get_and_verify_dtype( ...@@ -1087,9 +1088,11 @@ def _get_and_verify_dtype(
if torch_dtype != config_dtype: if torch_dtype != config_dtype:
if torch_dtype == torch.float32: if torch_dtype == torch.float32:
# Upcasting to float32 is allowed. # Upcasting to float32 is allowed.
logger.info("Upcasting %s to %s.", config_dtype, torch_dtype)
pass pass
elif config_dtype == torch.float32: elif config_dtype == torch.float32:
# Downcasting from float32 to float16 or bfloat16 is allowed. # Downcasting from float32 to float16 or bfloat16 is allowed.
logger.info("Downcasting %s to %s.", config_dtype, torch_dtype)
pass pass
else: else:
# Casting between float16 and bfloat16 is allowed with a warning. # Casting between float16 and bfloat16 is allowed with a warning.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment