Skip to content

GitLab

Unverified Commit 7a338db2 authored by EtienneDosSantos's avatar EtienneDosSantos Committed by GitHub
Browse files

Update functional.py

parent 2e46eefc
Show whitespace changes
Inline Side-by-side
Showing with 70 additions and 18 deletions
bitsandbytes/functional.py
View file @ 7a338db2
...@@ -28,57 +28,94 @@ if lib and lib.compiled_with_cuda: ...@@ -28,57 +28,94 @@ if lib and lib.compiled_with_cuda:
"""C FUNCTIONS FOR OPTIMIZERS""" """C FUNCTIONS FOR OPTIMIZERS"""
str2optimizer32bit = { str2optimizer32bit = {
"adagrad": ( "adagrad": (
lib.cadagrad32bit_grad_32, lib.cadagrad32bit_grad_fp32,
lib.cadagrad32bit_grad_16, lib.cadagrad32bit_grad_fp16,
), ),
"adam": ( "adam": (
lib.cadam32bit_grad_fp32, lib.cadam32bit_grad_fp32,
lib.cadam32bit_grad_fp16, lib.cadam32bit_grad_fp16,
lib.cadam32bit_grad_bf16, lib.cadam32bit_grad_bf16,
), ),
"pagedadam": (
lib.cpagedadam32bit_grad_fp32,
lib.cpagedadam32bit_grad_fp16,
lib.cpagedadam32bit_grad_bf16,
),
"adamw": (
lib.cadam32bit_grad_fp32,
lib.cadam32bit_grad_fp16,
lib.cadam32bit_grad_bf16,
),
"pagedadamw": (
lib.cpagedadam32bit_grad_fp32,
lib.cpagedadam32bit_grad_fp16,
lib.cpagedadam32bit_grad_bf16,
),
"lamb": ( "lamb": (
lib.cadam32bit_grad_fp32, lib.cadam32bit_grad_fp32,
lib.cadam32bit_grad_fp16, lib.cadam32bit_grad_fp16,
), ),
"lars": (
lib.clars32bit_grad_fp32,
lib.clars32bit_grad_fp16,
),
"lion": ( "lion": (
lib.clion32bit_grad_fp32, lib.clion32bit_grad_fp32,
lib.clion32bit_grad_fp16, lib.clion32bit_grad_fp16,
lib.clion32bit_grad_bf16, lib.clion32bit_grad_bf16,
), ),
"momentum": ( "momentum": (
lib.cmomentum32bit_grad_32, lib.cmomentum32bit_grad_fp32,
lib.cmomentum32bit_grad_16, lib.cmomentum32bit_grad_fp16,
), ),
"rmsprop": ( "rmsprop": (
lib.crmsprop32bit_grad_32, lib.crmsprop32bit_grad_fp32,
lib.crmsprop32bit_grad_16, lib.crmsprop32bit_grad_fp16,
), ),
} }
str2optimizer8bit = { str2optimizer8bit = {
"adagrad": (
lib.cadagrad8bit_grad_fp32,
lib.cadagrad8bit_grad_fp16,
),
"adam": ( "adam": (
lib.cadam_static_8bit_grad_32, lib.cadam_static_8bit_grad_fp32,
lib.cadam_static_8bit_grad_16, lib.cadam_static_8bit_grad_fp16,
),
"pagedadam": (
lib.cpagedadam8bit_grad_fp32,
lib.cpagedadam8bit_grad_fp16,
lib.cpagedadam8bit_grad_bf16,
),
"adamw": (
lib.cadam_static_8bit_grad_fp32,
lib.cadam_static_8bit_grad_fp16,
),
"pagedadamw": (
lib.cpagedadam8bit_grad_fp32,
lib.cpagedadam8bit_grad_fp16,
lib.cpagedadam8bit_grad_bf16,
), ),
"lamb": ( "lamb": (
lib.cadam_static_8bit_grad_32, lib.cadam_static_8bit_grad_fp32,
lib.cadam_static_8bit_grad_16, lib.cadam_static_8bit_grad_fp16,
), ),
"lars": ( "lars": (
lib.cmomentum_static_8bit_grad_32, lib.clars8bit_grad_fp32,
lib.cmomentum_static_8bit_grad_16, lib.clars8bit_grad_fp16,
), ),
"lion": ( "lion": (
lib.clion_static_8bit_grad_32, lib.clion_static_8bit_grad_fp32,
lib.clion_static_8bit_grad_16, lib.clion_static_8bit_grad_fp16,
), ),
"momentum": ( "momentum": (
lib.cmomentum_static_8bit_grad_32, lib.cmomentum_static_8bit_grad_fp32,
lib.cmomentum_static_8bit_grad_16, lib.cmomentum_static_8bit_grad_fp16,
), ),
"rmsprop": ( "rmsprop": (
lib.crmsprop_static_8bit_grad_32, lib.crmsprop_static_8bit_grad_fp32,
lib.crmsprop_static_8bit_grad_16, lib.crmsprop_static_8bit_grad_fp16,
), ),
} }
...@@ -92,6 +129,21 @@ if lib and lib.compiled_with_cuda: ...@@ -92,6 +129,21 @@ if lib and lib.compiled_with_cuda:
lib.cadam_8bit_blockwise_grad_fp16, lib.cadam_8bit_blockwise_grad_fp16,
lib.cadam_8bit_blockwise_grad_bf16, lib.cadam_8bit_blockwise_grad_bf16,
), ),
"pagedadam": (
lib.cpagedadam8bit_blockwise_fp32,
lib.cpagedadam8bit_blockwise_fp16,
lib.cpagedadam8bit_blockwise_bf16,
),
"adamw": (
lib.cadam_8bit_blockwise_grad_fp32,
lib.cadam_8bit_blockwise_grad_fp16,
lib.cadam_8bit_blockwise_grad_bf16,
),
"pagedadamw": (
lib.cpagedadam8bit_blockwise_fp32,
lib.cpagedadam8bit_blockwise_fp16,
lib.cpagedadam8bit_blockwise_bf16,
),
"lion": ( "lion": (
lib.clion_8bit_blockwise_grad_fp32, lib.clion_8bit_blockwise_grad_fp32,
lib.clion_8bit_blockwise_grad_fp16, lib.clion_8bit_blockwise_grad_fp16,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment