Skip to content

GitLab

Commit 5b92f9f0 authored by Scott Zhu's avatar Scott Zhu Committed by A. Unique TensorFlower
Browse files

Prepare for upcoming keras initializer change. 

PiperOrigin-RevId: 451475383
parent 7278d89b
Show whitespace changes
Inline Side-by-side
Showing with 4 additions and 3 deletions
official/projects/teams/teams_pretrainer.py
View file @ 5b92f9f0
...@@ -58,15 +58,16 @@ class ReplacedTokenDetectionHead(tf.keras.layers.Layer): ...@@ -58,15 +58,16 @@ class ReplacedTokenDetectionHead(tf.keras.layers.Layer):
intermediate_activation=self.activation, intermediate_activation=self.activation,
dropout_rate=self.hidden_cfg['dropout_rate'], dropout_rate=self.hidden_cfg['dropout_rate'],
attention_dropout_rate=self.hidden_cfg['attention_dropout_rate'], attention_dropout_rate=self.hidden_cfg['attention_dropout_rate'],
kernel_initializer=self.initializer, kernel_initializer=tf_utils.clone_initializer(self.initializer),
name='transformer/layer_%d_rtd' % i)) name='transformer/layer_%d_rtd' % i))
self.dense = tf.keras.layers.Dense( self.dense = tf.keras.layers.Dense(
self.hidden_size, self.hidden_size,
activation=self.activation, activation=self.activation,
kernel_initializer=self.initializer, kernel_initializer=tf_utils.clone_initializer(self.initializer),
name='transform/rtd_dense') name='transform/rtd_dense')
self.rtd_head = tf.keras.layers.Dense( self.rtd_head = tf.keras.layers.Dense(
units=1, kernel_initializer=self.initializer, units=1,
kernel_initializer=tf_utils.clone_initializer(self.initializer),
name='transform/rtd_head') name='transform/rtd_head')
if output not in ('predictions', 'logits'): if output not in ('predictions', 'logits'):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment