Skip to content

GitLab

Commit fbec2dbe authored by A. Unique TensorFlower's avatar A. Unique TensorFlower
Browse files

Internal change 

PiperOrigin-RevId: 313536026
parent fad183f8
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
official/nlp/optimization.py
View file @ fbec2dbe
...@@ -93,7 +93,7 @@ def create_optimizer(init_lr, ...@@ -93,7 +93,7 @@ def create_optimizer(init_lr,
beta_1=0.9, beta_1=0.9,
beta_2=0.999, beta_2=0.999,
epsilon=1e-6, epsilon=1e-6,
exclude_from_weight_decay=['layer_norm', 'bias']) exclude_from_weight_decay=['LayerNorm', 'layer_norm', 'bias'])
elif optimizer_type == 'lamb': elif optimizer_type == 'lamb':
logging.info('using Lamb optimizer') logging.info('using Lamb optimizer')
optimizer = tfa_optimizers.LAMB( optimizer = tfa_optimizers.LAMB(
...@@ -102,7 +102,7 @@ def create_optimizer(init_lr, ...@@ -102,7 +102,7 @@ def create_optimizer(init_lr,
beta_1=0.9, beta_1=0.9,
beta_2=0.999, beta_2=0.999,
epsilon=1e-6, epsilon=1e-6,
exclude_from_weight_decay=['layer_norm', 'bias']) exclude_from_weight_decay=['LayerNorm', 'layer_norm', 'bias'])
else: else:
raise ValueError('Unsupported optimizer type: ', optimizer_type) raise ValueError('Unsupported optimizer type: ', optimizer_type)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment