xlnet_modeling.py 46.5 KB
Newer Older
Frederick Liu's avatar
Frederick Liu committed
1
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Hongkun Yu's avatar
Hongkun Yu committed
2
3
4
5
6
7
8
9
10
11
12
13
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Frederick Liu's avatar
Frederick Liu committed
14

Hongkun Yu's avatar
Hongkun Yu committed
15
16
17
"""Keras layers of XLNet model in TF 2.0."""

import copy
Allen Wang's avatar
Allen Wang committed
18
import warnings
Hongkun Yu's avatar
Hongkun Yu committed
19
20

import tensorflow as tf
Allen Wang's avatar
Allen Wang committed
21
22

from official.nlp.modeling import networks
Hongkun Yu's avatar
Hongkun Yu committed
23
from official.nlp.xlnet import data_utils
Hongkun Yu's avatar
Hongkun Yu committed
24
25


A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
26
27
def gelu(x):
  return tf.keras.activations.gelu(x, approximate=True)
Hongkun Yu's avatar
Hongkun Yu committed
28
29


Allen Wang's avatar
Allen Wang committed
30
31
32
33
34
35
36
37
38
39
40
41
def _get_initializer(flags):
  """Get variable initializer."""
  if flags.init_method == "uniform":
    initializer = tf.keras.initializers.RandomUniform(
        minval=-flags.init_range, maxval=flags.init_range)
  elif flags.init_method == "normal":
    initializer = tf.keras.initializers.RandomNormal(stddev=flags.init_std)
  else:
    raise ValueError("Initializer {} not supported".format(flags.init_method))
  return initializer


Hongkun Yu's avatar
Hongkun Yu committed
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def rel_shift(x, klen=-1):
  """Performs relative shift to form the relative attention score."""
  x_size = tf.shape(x)

  x = tf.reshape(x, [x_size[1], x_size[0], x_size[2], x_size[3]])
  x = tf.slice(x, [1, 0, 0, 0], [-1, -1, -1, -1])
  x = tf.reshape(x, [x_size[0], x_size[1] - 1, x_size[2], x_size[3]])
  x = tf.slice(x, [0, 0, 0, 0], [-1, klen, -1, -1])

  return x


def _create_mask(qlen, mlen, dtype=tf.float32, same_length=False):
  """Creates attention mask when single-side context allowed only."""
  attn_mask = tf.ones([qlen, qlen], dtype=dtype)
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
57
58
  mask_u = tf.linalg.band_part(attn_mask, 0, -1)
  mask_dia = tf.linalg.band_part(attn_mask, 0, 0)
Hongkun Yu's avatar
Hongkun Yu committed
59
60
61
  attn_mask_pad = tf.zeros([qlen, mlen], dtype=dtype)
  ret = tf.concat([attn_mask_pad, mask_u - mask_dia], 1)
  if same_length:
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
62
    mask_l = tf.linalg.band_part(attn_mask, -1, 0)
Hongkun Yu's avatar
Hongkun Yu committed
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    ret = tf.concat([ret[:, :qlen] + mask_l - mask_dia, ret[:, qlen:]], 1)

  return ret


def _cache_mem(curr_out, prev_mem, mem_len, reuse_len=None):
  """cache hidden states into memory."""

  if mem_len is None or mem_len == 0:
    return None
  else:
    if reuse_len is not None and reuse_len > 0:
      curr_out = curr_out[:reuse_len]

    if prev_mem is None:
      new_mem = curr_out[-mem_len:]
    else:
      new_mem = tf.concat([prev_mem, curr_out], 0)[-mem_len:]

  return tf.keras.backend.stop_gradient(new_mem)


def is_special_none_tensor(tensor):
  """Checks if a tensor is a special None Tensor."""
  return tensor.shape.ndims == 0 and tensor.dtype == tf.int32


Allen Wang's avatar
Allen Wang committed
90
@tf.keras.utils.register_keras_serializable(package="Text")
Allen Wang's avatar
Allen Wang committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
class RelativePositionEncoding(tf.keras.layers.Layer):
  """Creates a relative positional encoding.

  This layer creates a relative positional encoding as described in
  "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context"
  (https://arxiv.org/abs/1901.02860).

  Rather than an absolute position embedding as in Transformer, this
  formulation represents position as the relative distance between tokens using
  sinusoidal positional embeddings.

  Note: This layer is currently experimental.

  Attributes:
    hidden_size: The dimensionality of the input embeddings.
  """

  def __init__(self, hidden_size, **kwargs):
    super(RelativePositionEncoding, self).__init__(**kwargs)
    self._hidden_size = hidden_size
    self._inv_freq = 1.0 / (10000.0**(
        tf.range(0, self._hidden_size, 2.0) / self._hidden_size))

  def call(self, pos_seq, batch_size=None):
    """Implements call() for the layer.

117
    Args:
Allen Wang's avatar
Allen Wang committed
118
119
120
121
122
123
124
125
126
      pos_seq: A 1-D `Tensor`
      batch_size: The optionally provided batch size that tiles the relative
        positional encoding.

    Returns:
      The relative positional encoding of shape:
        [len(pos_seq), batch_size, hidden_size] if batch_size is provided, else
        [len(pos_seq), 1, hidden_size].
    """
Allen Wang's avatar
Allen Wang committed
127
    sinusoid_input = tf.einsum("i,d->id", pos_seq, self._inv_freq)
Allen Wang's avatar
Allen Wang committed
128
    pos_emb = tf.concat([tf.sin(sinusoid_input), tf.cos(sinusoid_input)], -1)
Hongkun Yu's avatar
Hongkun Yu committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    pos_emb = pos_emb[:, None, :]

    if batch_size is not None:
      pos_emb = tf.tile(pos_emb, [1, batch_size, 1])
    return pos_emb


class RelativeAttention(tf.keras.layers.Layer):
  """Core calculations for relative attention."""

  def __init__(self, dropout_att, scale):
    super(RelativeAttention, self).__init__()
    self.scale = scale
    self.dropout_att = dropout_att

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""

    self.attention_probs_dropout = tf.keras.layers.Dropout(
        rate=self.dropout_att)

    super(RelativeAttention, self).build(unused_input_shapes)

Hongkun Yu's avatar
Hongkun Yu committed
152
153
  def call(self, q_head, k_head_h, v_head_h, k_head_r, seg_embed, seg_mat,
           r_w_bias, r_r_bias, r_s_bias, attn_mask):
Hongkun Yu's avatar
Hongkun Yu committed
154
155
156
    """Implements call() for the layer."""

    # content based attention score
Allen Wang's avatar
Allen Wang committed
157
    ac = tf.einsum("ibnd,jbnd->ijbn", q_head + r_w_bias, k_head_h)
Hongkun Yu's avatar
Hongkun Yu committed
158
159

    # position based attention score
Allen Wang's avatar
Allen Wang committed
160
    bd = tf.einsum("ibnd,jbnd->ijbn", q_head + r_r_bias, k_head_r)
Hongkun Yu's avatar
Hongkun Yu committed
161
162
163
164
165
166
    bd = rel_shift(bd, klen=tf.shape(ac)[1])

    # segment-based attention score
    if seg_mat is None:
      ef = 0
    else:
Allen Wang's avatar
Allen Wang committed
167
      ef = tf.einsum("ibnd,snd->isbn", q_head + r_s_bias, seg_embed)
Hongkun Yu's avatar
Hongkun Yu committed
168
169
170
171
172
      tgt_shape = tf.shape(bd)
      ef = tf.where(
          tf.broadcast_to(tf.expand_dims(seg_mat, 3), tgt_shape),
          tf.broadcast_to(ef[:, 1:, :, :], tgt_shape),
          tf.broadcast_to(ef[:, :1, :, :], tgt_shape))
Hongkun Yu's avatar
Hongkun Yu committed
173
174
175
176
177
178
179
180
181
182
183

    # merges attention scores and performs masking
    attn_score = (ac + bd + ef) * self.scale
    if attn_mask is not None:
      attn_score = attn_score - 1e30 * attn_mask

    # attention probability
    attn_prob = tf.nn.softmax(attn_score, 1)
    attn_prob = self.attention_probs_dropout(attn_prob)

    # attention output
Allen Wang's avatar
Allen Wang committed
184
    attn_vec = tf.einsum("ijbn,jbnd->ibnd", attn_prob, v_head_h)
Hongkun Yu's avatar
Hongkun Yu committed
185
186
187
188
189
190
191

    return attn_vec


class PositionwiseFF(tf.keras.layers.Layer):
  """Positionwise feed-forward layer."""

Hongkun Yu's avatar
Hongkun Yu committed
192
193
  def __init__(self, d_model, d_inner, dropout, kernel_initializer,
               activation_type, **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
194
195
196
197
198
199
200
201
202
    super(PositionwiseFF, self).__init__(**kwargs)
    self.d_model = d_model
    self.d_inner = d_inner
    self.dropout = dropout
    self.activation_type = activation_type
    self.kernel_initializer = kernel_initializer

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
Allen Wang's avatar
Allen Wang committed
203
    if self.activation_type == "relu":
Hongkun Yu's avatar
Hongkun Yu committed
204
      activation = tf.nn.relu
Allen Wang's avatar
Allen Wang committed
205
    elif self.activation_type == "gelu":
Hongkun Yu's avatar
Hongkun Yu committed
206
207
      activation = gelu
    else:
Allen Wang's avatar
Allen Wang committed
208
      raise (ValueError("Unsupported activation type {}".format(
Hongkun Yu's avatar
Hongkun Yu committed
209
210
211
212
213
214
          self.activation_type)))
    self.inner_projection_layer = (
        tf.keras.layers.Dense(
            units=self.d_inner,
            activation=activation,
            kernel_initializer=self.kernel_initializer,
Allen Wang's avatar
Allen Wang committed
215
            name="layer_1"))
Hongkun Yu's avatar
Hongkun Yu committed
216
217
218
219
    self.output_projection_layer = (
        tf.keras.layers.Dense(
            units=self.d_model,
            kernel_initializer=self.kernel_initializer,
Allen Wang's avatar
Allen Wang committed
220
            name="layer_2"))
Hongkun Yu's avatar
Hongkun Yu committed
221
    self.output_dropout = tf.keras.layers.Dropout(
Allen Wang's avatar
Allen Wang committed
222
        rate=self.dropout, name="drop_2")
Hongkun Yu's avatar
Hongkun Yu committed
223
224
    self.output_layer_norm = (
        tf.keras.layers.LayerNormalization(
Allen Wang's avatar
Allen Wang committed
225
            name="LayerNorm", axis=-1, epsilon=1e-12))
Hongkun Yu's avatar
Hongkun Yu committed
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    super(PositionwiseFF, self).build(unused_input_shapes)

  def call(self, inp):
    """Implements call() for the layer."""

    output = self.inner_projection_layer(inp)
    output = self.output_projection_layer(output)
    output = self.output_dropout(output)
    output = self.output_layer_norm(output + inp)
    return output


class EmbeddingLookup(tf.keras.layers.Layer):
  """Looks up words embeddings for id tensor."""

Hongkun Yu's avatar
Hongkun Yu committed
241
  def __init__(self, n_token, d_embed, initializer, **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
242
243
244
245
246
247
248
249
    super(EmbeddingLookup, self).__init__(**kwargs)
    self.n_token = n_token
    self.d_embed = d_embed
    self.initializer = initializer

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
    self.lookup_table = self.add_weight(
Allen Wang's avatar
Allen Wang committed
250
        "lookup_table",
Hongkun Yu's avatar
Hongkun Yu committed
251
252
253
254
255
256
257
        shape=[self.n_token, self.d_embed],
        initializer=self.initializer,
        dtype=self.dtype)

    super(EmbeddingLookup, self).build(unused_input_shapes)

  def call(self, inputs):
Hongkun Yu's avatar
Hongkun Yu committed
258
    return tf.nn.embedding_lookup(self.lookup_table, inputs)
Hongkun Yu's avatar
Hongkun Yu committed
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275


class RelativeMultiheadAttention(tf.keras.layers.Layer):
  """Multi-head attention with relative embedding."""

  def __init__(self, d_model, n_head, d_head, dropout, dropout_att,
               kernel_initializer, **kwargs):
    super(RelativeMultiheadAttention, self).__init__(**kwargs)
    self.d_model = d_model
    self.n_head = n_head
    self.d_head = d_head
    self.dropout = dropout
    self.dropout_att = dropout_att
    self.initializer = kernel_initializer

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
Hongkun Yu's avatar
Hongkun Yu committed
276
    self.scale = 1.0 / (self.d_head**0.5)
Hongkun Yu's avatar
Hongkun Yu committed
277
278

    self.output_layer_norm = tf.keras.layers.LayerNormalization(
Allen Wang's avatar
Allen Wang committed
279
        name="LayerNorm", axis=-1, epsilon=1e-12)
Hongkun Yu's avatar
Hongkun Yu committed
280
281

    self.kh_projection_layer = self.add_weight(
Allen Wang's avatar
Allen Wang committed
282
        "k/kernel",
Hongkun Yu's avatar
Hongkun Yu committed
283
284
285
        shape=[self.d_model, self.n_head, self.d_head],
        initializer=self.initializer)
    self.vh_projection_layer = self.add_weight(
Allen Wang's avatar
Allen Wang committed
286
        "v/kernel",
Hongkun Yu's avatar
Hongkun Yu committed
287
288
289
        shape=[self.d_model, self.n_head, self.d_head],
        initializer=self.initializer)
    self.kr_projection_layer = self.add_weight(
Allen Wang's avatar
Allen Wang committed
290
        "r/kernel",
Hongkun Yu's avatar
Hongkun Yu committed
291
292
293
        shape=[self.d_model, self.n_head, self.d_head],
        initializer=self.initializer)
    self.qh_projection_layer = self.add_weight(
Allen Wang's avatar
Allen Wang committed
294
        "q/kernel",
Hongkun Yu's avatar
Hongkun Yu committed
295
296
297
        shape=[self.d_model, self.n_head, self.d_head],
        initializer=self.initializer)

298
    self.relative_attention_layer = RelativeAttention(
Hongkun Yu's avatar
Hongkun Yu committed
299
300
301
        dropout_att=self.dropout_att, scale=self.scale)

    self.proj_o = self.add_weight(
Allen Wang's avatar
Allen Wang committed
302
        "o/kernel",
Hongkun Yu's avatar
Hongkun Yu committed
303
304
305
306
307
308
309
        shape=[self.d_model, self.n_head, self.d_head],
        initializer=self.initializer)

    self.attention_dropout = tf.keras.layers.Dropout(rate=self.dropout)

    super(RelativeMultiheadAttention, self).build(unused_input_shapes)

Hongkun Yu's avatar
Hongkun Yu committed
310
311
  def call(self, h, g, r, r_w_bias, r_r_bias, seg_mat, r_s_bias, seg_embed,
           attn_mask_h, attn_mask_g, mems, target_mapping):
Hongkun Yu's avatar
Hongkun Yu committed
312
313
314
315
316
317
318
319
    """Implements call() for the layer."""

    if mems is not None and mems.shape.ndims > 1:
      cat = tf.concat([mems, h], 0)
    else:
      cat = h

    # content heads
Allen Wang's avatar
Allen Wang committed
320
321
322
    q_head_h = tf.einsum("ibh,hnd->ibnd", h, self.qh_projection_layer)
    k_head_h = tf.einsum("ibh,hnd->ibnd", cat, self.kh_projection_layer)
    v_head_h = tf.einsum("ibh,hnd->ibnd", cat, self.vh_projection_layer)
Hongkun Yu's avatar
Hongkun Yu committed
323
324

    # positional heads
Allen Wang's avatar
Allen Wang committed
325
    k_head_r = tf.einsum("ibh,hnd->ibnd", r, self.kr_projection_layer)
Hongkun Yu's avatar
Hongkun Yu committed
326
327

    # core attention ops
Hongkun Yu's avatar
Hongkun Yu committed
328
329
330
331
    attn_vec_h = self.relative_attention_layer(q_head_h, k_head_h, v_head_h,
                                               k_head_r, seg_embed, seg_mat,
                                               r_w_bias, r_r_bias, r_s_bias,
                                               attn_mask_h)
Hongkun Yu's avatar
Hongkun Yu committed
332
333

    # post processing
Allen Wang's avatar
Allen Wang committed
334
    output_h = tf.einsum("ibnd,hnd->ibh", attn_vec_h, self.proj_o)
335
336
    output_h = self.attention_dropout(output_h)
    output_h = self.output_layer_norm(output_h + h)
Hongkun Yu's avatar
Hongkun Yu committed
337

338
339
340
    output_g = None
    if g is not None:  # enable two-stream attention
      # g-stream
Allen Wang's avatar
Allen Wang committed
341
      q_head_g = tf.einsum("ibh,hnd->ibnd", g, self.qh_projection_layer)
342
      if target_mapping is not None:
Allen Wang's avatar
Allen Wang committed
343
        q_head_g = tf.einsum("mbnd,mlb->lbnd", q_head_g, target_mapping)
Hongkun Yu's avatar
Hongkun Yu committed
344
345
346
347
        attn_vec_g = self.relative_attention_layer(q_head_g, k_head_h, v_head_h,
                                                   k_head_r, seg_embed, seg_mat,
                                                   r_w_bias, r_r_bias, r_s_bias,
                                                   attn_mask_g)
Allen Wang's avatar
Allen Wang committed
348
        attn_vec_g = tf.einsum("lbnd,mlb->mbnd", attn_vec_g, target_mapping)
Hongkun Yu's avatar
Hongkun Yu committed
349

350
      else:
Hongkun Yu's avatar
Hongkun Yu committed
351
352
353
354
        attn_vec_g = self.relative_attention_layer(q_head_g, k_head_h, v_head_h,
                                                   k_head_r, seg_embed, seg_mat,
                                                   r_w_bias, r_r_bias, r_s_bias,
                                                   attn_mask_g)
Hongkun Yu's avatar
Hongkun Yu committed
355

356
      # post processing
Allen Wang's avatar
Allen Wang committed
357
      output_g = tf.einsum("ibnd,hnd->ibh", attn_vec_g, self.proj_o)
358
359
360
361
      output_g = self.attention_dropout(output_g)
      output_g = self.output_layer_norm(output_g + g)

    return (output_h, output_g)
Hongkun Yu's avatar
Hongkun Yu committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385


class TransformerXLModel(tf.keras.layers.Layer):
  """Defines a Transformer-XL computation graph with additional support for XLNet."""

  def __init__(self,
               n_token,
               n_layer,
               d_model,
               n_head,
               d_head,
               d_inner,
               dropout,
               dropout_att,
               attn_type,
               bi_data,
               is_training,
               initializer,
               mem_len=None,
               same_length=False,
               clamp_len=-1,
               untie_r=False,
               use_tpu=True,
               reuse_len=None,
Allen Wang's avatar
Allen Wang committed
386
               ff_activation="relu",
Hongkun Yu's avatar
Hongkun Yu committed
387
               use_cls_mask=False,
Hongkun Yu's avatar
Hongkun Yu committed
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
               **kwargs):
    """Initializes TransformerXLModel.

    Args:
      n_token: int, the number of tokens in vocabulary.
      n_layer: int, the number of layers.
      d_model: int, the hidden size.
      n_head: int, the number of attention heads.
      d_head: int, the dimension size of each attention head.
      d_inner: int, the hidden size in feed-forward layers.
      dropout: float, dropout rate.
      dropout_att: float, dropout rate on attention probabilities.
      attn_type: str, "uni" or "bi".
      bi_data: bool, whether to use bidirectional input pipeline. Usually set to
        True during pretraining and False during finetuning.
      is_training: bool, whether in training mode.
      initializer: A tf initializer.
      mem_len: int, the number of tokens to cache.
      same_length: bool, whether to use the same attention length for each
        token.
      clamp_len: int, clamp all relative distances larger than clamp_len. -1
        means no clamping.
      untie_r: bool, whether to untie the biases in attention.
      use_tpu: bool, whether TPUs are used.
      reuse_len: int, the number of tokens in the currect batch to be cached and
        reused in the future.
      ff_activation: str, "relu" or "gelu".
Hongkun Yu's avatar
Hongkun Yu committed
415
      use_cls_mask: bool, whether to introduce cls mask.
Hongkun Yu's avatar
Hongkun Yu committed
416
417
418
419
      **kwargs: Other parameters.
    """

    super(TransformerXLModel, self).__init__(**kwargs)
Allen Wang's avatar
Allen Wang committed
420
421
422
    warnings.warn(
        "`TransformerXLModel` is deprecated, please use `XLNetBase` instead",
        DeprecationWarning, stacklevel=2)
Hongkun Yu's avatar
Hongkun Yu committed
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442

    self.n_token = n_token
    self.initializer = initializer
    self.attn_type = attn_type
    self.n_layer = n_layer
    self.d_model = d_model
    self.n_head = n_head
    self.d_head = d_head
    self.d_inner = d_inner
    self.ff_activation = ff_activation
    self.untie_r = untie_r
    self.use_tpu = use_tpu
    self.dropout = dropout
    self.dropout_att = dropout_att

    self.mem_len = mem_len
    self.reuse_len = reuse_len
    self.bi_data = bi_data
    self.clamp_len = clamp_len
    self.same_length = same_length
Hongkun Yu's avatar
Hongkun Yu committed
443
    self.use_cls_mask = use_cls_mask
Hongkun Yu's avatar
Hongkun Yu committed
444
445
446

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
Hongkun Yu's avatar
Hongkun Yu committed
447
    self.tf_float = tf.float32
Hongkun Yu's avatar
Hongkun Yu committed
448

Hongkun Yu's avatar
Hongkun Yu committed
449
450
451
452
453
    self.embedding_lookup = EmbeddingLookup(
        n_token=self.n_token,
        d_embed=self.d_model,
        initializer=self.initializer,
        dtype=self.tf_float,
Allen Wang's avatar
Allen Wang committed
454
        name="word_embedding")
Hongkun Yu's avatar
Hongkun Yu committed
455
456
457
458
459
460
461

    self.h_dropout = tf.keras.layers.Dropout(rate=self.dropout)
    self.g_dropout = tf.keras.layers.Dropout(rate=self.dropout)

    if self.untie_r:
      self.r_w_bias = (
          self.add_weight(
Allen Wang's avatar
Allen Wang committed
462
              "r_w_bias",
Hongkun Yu's avatar
Hongkun Yu committed
463
464
465
466
467
              shape=[self.n_layer, self.n_head, self.d_head],
              dtype=self.tf_float,
              initializer=self.initializer))
      self.r_r_bias = (
          self.add_weight(
Allen Wang's avatar
Allen Wang committed
468
              "r_r_bias",
Hongkun Yu's avatar
Hongkun Yu committed
469
470
471
472
473
              shape=[self.n_layer, self.n_head, self.d_head],
              dtype=self.tf_float,
              initializer=self.initializer))
      self.r_s_bias = (
          self.add_weight(
Allen Wang's avatar
Allen Wang committed
474
              "r_s_bias",
Hongkun Yu's avatar
Hongkun Yu committed
475
476
477
478
479
480
              shape=[self.n_layer, self.n_head, self.d_head],
              dtype=self.tf_float,
              initializer=self.initializer))
    else:
      self.r_w_bias = (
          self.add_weight(
Allen Wang's avatar
Allen Wang committed
481
              "r_w_bias",
Hongkun Yu's avatar
Hongkun Yu committed
482
483
484
485
486
              shape=[self.n_head, self.d_head],
              dtype=self.tf_float,
              initializer=self.initializer))
      self.r_r_bias = (
          self.add_weight(
Allen Wang's avatar
Allen Wang committed
487
              "r_r_bias",
Hongkun Yu's avatar
Hongkun Yu committed
488
489
490
491
492
              shape=[self.n_head, self.d_head],
              dtype=self.tf_float,
              initializer=self.initializer))
      self.r_s_bias = (
          self.add_weight(
Allen Wang's avatar
Allen Wang committed
493
              "r_s_bias", [self.n_head, self.d_head],
Hongkun Yu's avatar
Hongkun Yu committed
494
495
496
497
              dtype=self.tf_float,
              initializer=self.initializer))

    self.seg_embed = self.add_weight(
Allen Wang's avatar
Allen Wang committed
498
        "seg_embed", [self.n_layer, 2, self.n_head, self.d_head],
Hongkun Yu's avatar
Hongkun Yu committed
499
500
        dtype=self.tf_float,
        initializer=self.initializer)
Hongkun Yu's avatar
Hongkun Yu committed
501

Hongkun Yu's avatar
Hongkun Yu committed
502
    self.mask_emb = self.add_weight(
Allen Wang's avatar
Allen Wang committed
503
        "mask_emb/mask_emb", shape=[1, 1, self.d_model], dtype=self.tf_float)
Hongkun Yu's avatar
Hongkun Yu committed
504
505

    self.emb_dropout = tf.keras.layers.Dropout(rate=self.dropout)
Allen Wang's avatar
Allen Wang committed
506
507
    self.fwd_position_embedding = RelativePositionEncoding(self.d_model)
    self.bwd_position_embedding = RelativePositionEncoding(self.d_model)
Hongkun Yu's avatar
Hongkun Yu committed
508
509
510
511
512
513
514
515
516
517
518
519

    self.rel_multihead_layers = []
    self.h_positionwise_ffn_layers = []
    for i in range(self.n_layer):
      self.rel_multihead_layers.append(
          RelativeMultiheadAttention(
              d_model=self.d_model,
              dropout=self.dropout,
              n_head=self.n_head,
              d_head=self.d_head,
              dropout_att=self.dropout_att,
              kernel_initializer=self.initializer,
Allen Wang's avatar
Allen Wang committed
520
              name="layer_%d/rel_attn" % (i)))
Hongkun Yu's avatar
Hongkun Yu committed
521
522
523
524
525
526
      self.h_positionwise_ffn_layers.append(
          PositionwiseFF(
              d_model=self.d_model,
              d_inner=self.d_inner,
              dropout=self.dropout,
              kernel_initializer=self.initializer,
Hongkun Yu's avatar
Hongkun Yu committed
527
              activation_type=self.ff_activation,
Allen Wang's avatar
Allen Wang committed
528
              name="layer_%d/ff" % (i)))
Hongkun Yu's avatar
Hongkun Yu committed
529
530
531
532
533
534
535
536
537
538
539
540

    self.output_dropout = tf.keras.layers.Dropout(rate=self.dropout)

    super(TransformerXLModel, self).build(unused_input_shapes)

  def __call__(self,
               inp_k,
               seg_id=None,
               input_mask=None,
               mems=None,
               perm_mask=None,
               target_mapping=None,
541
542
               inp_q=None,
               **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
543
544
    # Uses dict to feed inputs into call() in order to keep mems as a python
    # list.
Hongkun Yu's avatar
Hongkun Yu committed
545
    inputs = {
Allen Wang's avatar
Allen Wang committed
546
547
548
549
550
551
552
        "inp_k": inp_k,
        "seg_id": seg_id,
        "input_mask": input_mask,
        "mems": mems,
        "perm_mask": perm_mask,
        "target_mapping": target_mapping,
        "inp_q": inp_q
Hongkun Yu's avatar
Hongkun Yu committed
553
    }
554
    return super(TransformerXLModel, self).__call__(inputs, **kwargs)
Hongkun Yu's avatar
Hongkun Yu committed
555
556
557

  def call(self, inputs):
    """Implements call() for the layer."""
Allen Wang's avatar
Allen Wang committed
558
559
560
561
562
563
564
    inp_k = inputs["inp_k"]
    seg_id = inputs["seg_id"]
    input_mask = inputs["input_mask"]
    mems = inputs["mems"]
    perm_mask = inputs["perm_mask"]
    target_mapping = inputs["target_mapping"]
    inp_q = inputs["inp_q"]
Hongkun Yu's avatar
Hongkun Yu committed
565
566
567
568
569
570
571
572
573
574
575
576

    new_mems = []

    bsz = tf.shape(inp_k)[1]

    qlen = inp_k.shape.as_list()[0]

    mlen = mems[0].shape.as_list()[0] if mems is not None else 0
    klen = mlen + qlen

    ##### Attention mask
    # causal attention mask
Allen Wang's avatar
Allen Wang committed
577
    if self.attn_type == "uni":
Hongkun Yu's avatar
Hongkun Yu committed
578
579
580
      attn_mask = _create_mask(qlen, mlen, self.tf_float, self.same_length)
      # pylint: enable=protected-access
      attn_mask = attn_mask[:, :, None, None]
Allen Wang's avatar
Allen Wang committed
581
    elif self.attn_type == "bi":
Hongkun Yu's avatar
Hongkun Yu committed
582
583
      attn_mask = None
    else:
Allen Wang's avatar
Allen Wang committed
584
      raise ValueError("Unsupported attention type: {}".format(self.attn_type))
Hongkun Yu's avatar
Hongkun Yu committed
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611

    # data mask: input mask & perm mask
    if input_mask is not None and perm_mask is not None:
      data_mask = input_mask[None] + perm_mask

    elif input_mask is not None and perm_mask is None:
      data_mask = input_mask[None]
    elif input_mask is None and perm_mask is not None:
      data_mask = perm_mask
    else:
      data_mask = None

    if data_mask is not None:
      # all mems can be attended to
      mems_mask = tf.zeros([tf.shape(data_mask)[0], mlen, bsz],
                           dtype=self.tf_float)
      data_mask = tf.concat([mems_mask, data_mask], 1)
      if attn_mask is None:
        attn_mask = data_mask[:, :, :, None]
      else:
        attn_mask += data_mask[:, :, :, None]

    if attn_mask is not None:
      attn_mask = tf.cast(attn_mask > 0, dtype=self.tf_float)

    if attn_mask is not None:
      non_tgt_mask = -tf.eye(qlen, dtype=self.tf_float)
Hongkun Yu's avatar
Hongkun Yu committed
612
613
614
615
      non_tgt_mask = tf.concat(
          [tf.zeros([qlen, mlen], dtype=self.tf_float), non_tgt_mask], axis=-1)
      non_tgt_mask = tf.cast(
          (attn_mask + non_tgt_mask[:, :, None, None]) > 0, dtype=self.tf_float)
Hongkun Yu's avatar
Hongkun Yu committed
616
617
618
    else:
      non_tgt_mask = None

Hongkun Yu's avatar
Hongkun Yu committed
619
    word_emb_k = self.embedding_lookup(inp_k)
Hongkun Yu's avatar
Hongkun Yu committed
620
621
622
623
624
625
626
627
628
629

    if inp_q is not None:
      if target_mapping is not None:
        word_emb_q = tf.tile(self.mask_emb,
                             [tf.shape(target_mapping)[0], bsz, 1])
      else:
        inp_q_ext = inp_q[:, :, None]
        word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k

    output_h = self.h_dropout(word_emb_k)
630
    output_g = None
Hongkun Yu's avatar
Hongkun Yu committed
631
632
633
634
635
636
637
638
639
640
    if inp_q is not None:
      output_g = self.g_dropout(word_emb_q)

    ##### Segment embedding
    if seg_id is not None:

      # Convert `seg_id` to one-hot `seg_mat`

      mem_pad = tf.zeros([mlen, bsz], dtype=tf.int32)

Hongkun Yu's avatar
Hongkun Yu committed
641
      cat_id = tf.concat([mem_pad, seg_id], 0)
Hongkun Yu's avatar
Hongkun Yu committed
642

Hongkun Yu's avatar
Hongkun Yu committed
643
644
645
646
647
648
649
650
651
652
      if self.use_cls_mask:
        # `1` indicates not in the same segment [qlen x klen x bsz]
        # seg_id: [qlen x bsz] & cat_id: [klen x bsz]
        cls_mat = tf.logical_or(
            tf.equal(seg_id, tf.constant([data_utils.SEG_ID_CLS]))[:, None],
            tf.equal(cat_id, tf.constant([data_utils.SEG_ID_CLS]))[None, :])
        seg_mat = tf.equal(seg_id[:, None], cat_id[None, :])
        seg_mat = tf.logical_or(cls_mat, seg_mat)
      else:
        seg_mat = tf.logical_not(tf.equal(seg_id[:, None], cat_id[None, :]))
Hongkun Yu's avatar
Hongkun Yu committed
653
654
655
656
657
658
659
660
    else:
      seg_mat = None

    dtype = self.tf_float
    freq_seq = tf.range(0, self.d_model, 2.0)
    if dtype is not None and dtype != tf.float32:
      freq_seq = tf.cast(freq_seq, dtype=self.dtype)

Allen Wang's avatar
Allen Wang committed
661
    if self.attn_type == "bi":
Hongkun Yu's avatar
Hongkun Yu committed
662
      beg, end = klen, -qlen
Allen Wang's avatar
Allen Wang committed
663
    elif self.attn_type == "uni":
Hongkun Yu's avatar
Hongkun Yu committed
664
665
      beg, end = klen, -1
    else:
Allen Wang's avatar
Allen Wang committed
666
      raise ValueError("Unknown `attn_type` {}.".format(self.attn_type))
Hongkun Yu's avatar
Hongkun Yu committed
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682

    if self.bi_data:
      fwd_pos_seq = tf.range(beg, end, -1.0)
      bwd_pos_seq = tf.range(-beg, -end, 1.0)

      if dtype is not None and dtype != tf.float32:
        fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype)
        bwd_pos_seq = tf.cast(bwd_pos_seq, dtype=dtype)

      if self.clamp_len > 0:
        fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len,
                                       self.clamp_len)
        bwd_pos_seq = tf.clip_by_value(bwd_pos_seq, -self.clamp_len,
                                       self.clamp_len)

      if bsz is not None:
Hongkun Yu's avatar
Hongkun Yu committed
683
684
        fwd_pos_emb = self.fwd_position_embedding(fwd_pos_seq, bsz // 2)
        bwd_pos_emb = self.bwd_position_embedding(bwd_pos_seq, bsz // 2)
Hongkun Yu's avatar
Hongkun Yu committed
685
686
687
688
689
690
691
692
693
694
      else:
        fwd_pos_emb = self.fwd_position_embedding(fwd_pos_seq, None)
        bwd_pos_emb = self.bwd_position_embedding(bwd_pos_seq, None)

      pos_emb = tf.concat([fwd_pos_emb, bwd_pos_emb], axis=1)
    else:
      fwd_pos_seq = tf.range(beg, end, -1.0)
      if dtype is not None and dtype != tf.float32:
        fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype)
      if self.clamp_len > 0:
Hongkun Yu's avatar
Hongkun Yu committed
695
696
        fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len,
                                       self.lamp_len)
Hongkun Yu's avatar
Hongkun Yu committed
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717

      pos_emb = self.fwd_position_embedding(fwd_pos_seq, bsz)

    pos_emb = self.emb_dropout(pos_emb)

    if mems is None:
      mems = [None] * self.n_layer
    for i in range(self.n_layer):
      # cache new mems
      new_mems.append(
          _cache_mem(output_h, mems[i], self.mem_len, self.reuse_len))
      # pylint: enable=protected-access

      # segment bias
      if seg_id is None:
        r_s_bias_i = None
        seg_embed_i = None
      else:
        r_s_bias_i = self.r_s_bias if not self.untie_r else self.r_s_bias[i]
        seg_embed_i = self.seg_embed[i]

718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
      ffn_layer = self.h_positionwise_ffn_layers[i]
      attention_layer = self.rel_multihead_layers[i]
      output_h, output_g = attention_layer(
          h=output_h,
          g=output_g,
          r=pos_emb,
          r_w_bias=self.r_w_bias if not self.untie_r else self.r_w_bias[i],
          r_r_bias=self.r_r_bias if not self.untie_r else self.r_r_bias[i],
          seg_mat=seg_mat,
          r_s_bias=r_s_bias_i,
          seg_embed=seg_embed_i,
          attn_mask_h=non_tgt_mask,
          attn_mask_g=attn_mask,
          mems=mems[i],
          target_mapping=target_mapping)
      output_h = ffn_layer(output_h)
      if output_g is not None:
        output_g = ffn_layer(output_g)
Hongkun Yu's avatar
Hongkun Yu committed
736
737

    if inp_q is not None:
Hongkun Yu's avatar
Hongkun Yu committed
738
      output = output_g
Hongkun Yu's avatar
Hongkun Yu committed
739
    else:
Hongkun Yu's avatar
Hongkun Yu committed
740
      output = output_h
Hongkun Yu's avatar
Hongkun Yu committed
741
742
743
744
745
746
747
748
749
750
751

    return output, new_mems, None


class PretrainingXLNetModel(tf.keras.Model):
  """XLNet keras model combined with pretraining LM loss layer.

  See the original paper: https://arxiv.org/pdf/1906.08237.pdf

  """

Allen Wang's avatar
Allen Wang committed
752
753
  def __init__(self, use_proj, xlnet_config, run_config, use_legacy_mask=True,
               **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
754
755
756
757
    super(PretrainingXLNetModel, self).__init__(**kwargs)
    self.run_config = run_config
    self.initializer = _get_initializer(run_config)
    self.xlnet_config = copy.deepcopy(xlnet_config)
Allen Wang's avatar
Allen Wang committed
758
    self._use_legacy_mask = use_legacy_mask
Hongkun Yu's avatar
Hongkun Yu committed
759

Allen Wang's avatar
Allen Wang committed
760
761
    self.xlnet_model = networks.XLNetBase(
        vocab_size=self.xlnet_config.n_token,
Hongkun Yu's avatar
Hongkun Yu committed
762
        initializer=self.initializer,
Allen Wang's avatar
Allen Wang committed
763
764
765
766
767
768
769
770
771
772
773
774
775
        attention_type="bi",
        num_layers=self.xlnet_config.n_layer,
        hidden_size=self.xlnet_config.d_model,
        num_attention_heads=self.xlnet_config.n_head,
        head_size=self.xlnet_config.d_head,
        inner_size=self.xlnet_config.d_inner,
        two_stream=True,
        tie_attention_biases=not self.xlnet_config.untie_r,
        inner_activation=self.xlnet_config.ff_activation,
        dropout_rate=self.run_config.dropout,
        attention_dropout_rate=self.run_config.dropout_att,
        memory_length=self.run_config.mem_len,
        reuse_length=self.run_config.reuse_len,
Hongkun Yu's avatar
Hongkun Yu committed
776
        bi_data=self.run_config.bi_data,
Allen Wang's avatar
Allen Wang committed
777
        clamp_length=self.run_config.clamp_len,
Hongkun Yu's avatar
Hongkun Yu committed
778
        use_cls_mask=self.run_config.use_cls_mask,
Allen Wang's avatar
Allen Wang committed
779
780
        name="xlnet_model")

Hongkun Yu's avatar
Hongkun Yu committed
781
    self.lmloss_layer = LMLossLayer(
Allen Wang's avatar
Allen Wang committed
782
783
        vocab_size=self.xlnet_config.n_token,
        hidden_size=self.xlnet_config.d_model,
Hongkun Yu's avatar
Hongkun Yu committed
784
785
786
        initializer=self.initializer,
        tie_weight=True,
        bi_data=self.run_config.bi_data,
Allen Wang's avatar
Allen Wang committed
787
        use_one_hot=self.run_config.use_tpu,
Hongkun Yu's avatar
Hongkun Yu committed
788
        use_proj=use_proj,
Allen Wang's avatar
Allen Wang committed
789
        name="lm_loss")
Hongkun Yu's avatar
Hongkun Yu committed
790
791
792
793

  def call(self, features):
    """Implements call() for the layer."""

Allen Wang's avatar
Allen Wang committed
794
795
796
    input_ids = features["input_ids"]
    masked_tokens = features["input_q"]
    seg_ids = features["seg_id"]
Allen Wang's avatar
Allen Wang committed
797
    if self._use_legacy_mask:
798
799
      # Legacy input mask assumes `real` values are 0 and `padding`
      # values are 1.
Allen Wang's avatar
Allen Wang committed
800
801
802
      perm_mask = 1 - features["perm_mask"]
    else:
      perm_mask = features["perm_mask"]
Allen Wang's avatar
Allen Wang committed
803
    target_mapping = features["target_mapping"]
Hongkun Yu's avatar
Hongkun Yu committed
804
805

    # target for LM loss
Allen Wang's avatar
Allen Wang committed
806
    target = features["target"]
Hongkun Yu's avatar
Hongkun Yu committed
807
808

    # target mask for LM loss
Allen Wang's avatar
Allen Wang committed
809
    tgt_mask = features["target_mask"]
Hongkun Yu's avatar
Hongkun Yu committed
810

Allen Wang's avatar
Allen Wang committed
811
    mems = features.get("mems", None)
Hongkun Yu's avatar
Hongkun Yu committed
812

Allen Wang's avatar
Allen Wang committed
813
814
815
    model_output, self.new_mems = self.xlnet_model(
        input_ids=input_ids,
        segment_ids=seg_ids,
Hongkun Yu's avatar
Hongkun Yu committed
816
        input_mask=None,
Allen Wang's avatar
Allen Wang committed
817
818
        state=mems,
        permutation_mask=perm_mask,
Hongkun Yu's avatar
Hongkun Yu committed
819
        target_mapping=target_mapping,
Allen Wang's avatar
Allen Wang committed
820
        masked_tokens=masked_tokens)
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
821
    lm_loss, _ = self.lmloss_layer(
Allen Wang's avatar
Allen Wang committed
822
        hidden=model_output,
Hongkun Yu's avatar
Hongkun Yu committed
823
        target=target,
Allen Wang's avatar
Allen Wang committed
824
        lookup_table=self.xlnet_model.get_embedding_lookup_table(),
Hongkun Yu's avatar
Hongkun Yu committed
825
826
        target_mask=tgt_mask)
    self.add_loss(lm_loss)
Allen Wang's avatar
Allen Wang committed
827
    return self.new_mems, model_output
Hongkun Yu's avatar
Hongkun Yu committed
828
829
830
831
832
833
834
835
836


class ClassificationXLNetModel(tf.keras.Model):
  """XLNet keras model combined with classification loss layer.

  See the original paper: https://arxiv.org/pdf/1906.08237.pdf

  """

Allen Wang's avatar
Allen Wang committed
837
838
  def __init__(self, xlnet_config, run_config, n_class, summary_type,
               use_legacy_mask=True, **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
839
    super(ClassificationXLNetModel, self).__init__(**kwargs)
Allen Wang's avatar
Allen Wang committed
840
841
842
    warnings.warn(
        "`ClassificationXLNetModel` is deprecated, please use `XLNetClassifier`"
        "instead.", DeprecationWarning, stacklevel=2)
Hongkun Yu's avatar
Hongkun Yu committed
843
844
845
    self.run_config = run_config
    self.initializer = _get_initializer(run_config)
    self.xlnet_config = copy.deepcopy(xlnet_config)
Allen Wang's avatar
Allen Wang committed
846
    self._use_legacy_mask = use_legacy_mask
Hongkun Yu's avatar
Hongkun Yu committed
847

Allen Wang's avatar
Allen Wang committed
848
849
    self.xlnet_model = networks.XLNetBase(
        vocab_size=self.xlnet_config.n_token,
Hongkun Yu's avatar
Hongkun Yu committed
850
        initializer=self.initializer,
Allen Wang's avatar
Allen Wang committed
851
852
853
854
855
856
857
858
859
860
861
862
863
        attention_type="bi",
        num_layers=self.xlnet_config.n_layer,
        hidden_size=self.xlnet_config.d_model,
        num_attention_heads=self.xlnet_config.n_head,
        head_size=self.xlnet_config.d_head,
        inner_size=self.xlnet_config.d_inner,
        two_stream=False,
        tie_attention_biases=not self.xlnet_config.untie_r,
        inner_activation=self.xlnet_config.ff_activation,
        dropout_rate=self.run_config.dropout,
        attention_dropout_rate=self.run_config.dropout_att,
        memory_length=self.run_config.mem_len,
        reuse_length=self.run_config.reuse_len,
Hongkun Yu's avatar
Hongkun Yu committed
864
        bi_data=self.run_config.bi_data,
Allen Wang's avatar
Allen Wang committed
865
866
867
        clamp_length=self.run_config.clamp_len,
        use_cls_mask=False,
        name="xlnet_model")
Hongkun Yu's avatar
Hongkun Yu committed
868
869

    self.summarization_layer = Summarization(
Allen Wang's avatar
Allen Wang committed
870
871
872
873
874
        hidden_size=self.xlnet_config.d_model,
        num_attention_heads=self.xlnet_config.n_head,
        head_size=self.xlnet_config.d_head,
        dropout_rate=self.run_config.dropout,
        attention_dropout_rate=self.run_config.dropout_att,
Hongkun Yu's avatar
Hongkun Yu committed
875
876
        initializer=self.initializer,
        use_proj=True,
Hongkun Yu's avatar
Hongkun Yu committed
877
        summary_type=summary_type,
Allen Wang's avatar
Allen Wang committed
878
        name="sequence_summary")
Hongkun Yu's avatar
Hongkun Yu committed
879
880

    self.cl_loss_layer = ClassificationLossLayer(
Allen Wang's avatar
Allen Wang committed
881
        n_class=n_class, initializer=self.initializer, name="classification")
Hongkun Yu's avatar
Hongkun Yu committed
882
883
884

  def call(self, features):
    """Implements call() for the layer."""
Allen Wang's avatar
Allen Wang committed
885
    batch_size_per_core = tf.shape(features["input_ids"])[0]
Hongkun Yu's avatar
Hongkun Yu committed
886

Allen Wang's avatar
Allen Wang committed
887
888
    input_ids = features["input_ids"]
    segment_ids = features["segment_ids"]
Allen Wang's avatar
Allen Wang committed
889
    if self._use_legacy_mask:
890
891
      # Legacy input mask assumes `real` values are 0 and `padding`
      # values are 1.
Allen Wang's avatar
Allen Wang committed
892
893
894
      input_mask = 1 - features["input_mask"]
    else:
      input_mask = features["input_mask"]
Hongkun Yu's avatar
Hongkun Yu committed
895

Allen Wang's avatar
Allen Wang committed
896
    label = tf.reshape(features["label_ids"], [batch_size_per_core])
Hongkun Yu's avatar
Hongkun Yu committed
897

Allen Wang's avatar
Allen Wang committed
898
    mems = features.get("mems", None)
Hongkun Yu's avatar
Hongkun Yu committed
899

Allen Wang's avatar
Allen Wang committed
900
901
    attention_output, new_mems = (
        self.xlnet_model(input_ids, segment_ids, input_mask, mems))
Hongkun Yu's avatar
Hongkun Yu committed
902

Allen Wang's avatar
Allen Wang committed
903
    summary = self.summarization_layer(attention_output)
Hongkun Yu's avatar
Hongkun Yu committed
904
    per_example_loss, logits = self.cl_loss_layer(hidden=summary, labels=label)
Hongkun Yu's avatar
Hongkun Yu committed
905
    self.add_loss(tf.keras.backend.mean(per_example_loss))
Hongkun Yu's avatar
Hongkun Yu committed
906
    return new_mems, logits
Hongkun Yu's avatar
Hongkun Yu committed
907
908
909
910
911


class LMLossLayer(tf.keras.layers.Layer):
  """Layer computing cross entropy loss for language modeling."""

Hongkun Yu's avatar
Hongkun Yu committed
912
  def __init__(self,
Allen Wang's avatar
Allen Wang committed
913
914
               vocab_size,
               hidden_size,
Hongkun Yu's avatar
Hongkun Yu committed
915
916
917
               initializer,
               tie_weight=False,
               bi_data=True,
Allen Wang's avatar
Allen Wang committed
918
               use_one_hot=False,
Hongkun Yu's avatar
Hongkun Yu committed
919
920
               use_proj=False,
               **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
921
922
923
    """Constructs LMLoss layer.

    Args:
Allen Wang's avatar
Allen Wang committed
924
925
      vocab_size: Number of tokens in vocabulary.
      hidden_size: The dimension of model hidden state.
Hongkun Yu's avatar
Hongkun Yu committed
926
927
      initializer: Initializer used for parameters.
      tie_weight: Whether to share weights between embedding lookup layer and
Hongkun Yu's avatar
Hongkun Yu committed
928
929
930
        next-token prediction layer.
      bi_data: Whether to use bidirectional input pipeline. Usually set to True
        during pretraining and False during finetuning.
Allen Wang's avatar
Allen Wang committed
931
932
      use_one_hot: bool, whether to use one hot encodings. This should be used
        when TPUs are used.
Hongkun Yu's avatar
Hongkun Yu committed
933
      use_proj: bool, whether to add a projection layer before LM prediction.
Hongkun Yu's avatar
Hongkun Yu committed
934
935
936
      **kwargs: Other parameters.
    """
    super(LMLossLayer, self).__init__(**kwargs)
Allen Wang's avatar
Allen Wang committed
937
938
    self.vocab_size = vocab_size
    self.hidden_size = hidden_size
Hongkun Yu's avatar
Hongkun Yu committed
939
940
941
942
    self.initializer = initializer

    self.tie_weight = tie_weight
    self.bi_data = bi_data
Allen Wang's avatar
Allen Wang committed
943
    self.use_one_hot = use_one_hot
Hongkun Yu's avatar
Hongkun Yu committed
944
    self.use_proj = use_proj
Hongkun Yu's avatar
Hongkun Yu committed
945
946
947

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
Hongkun Yu's avatar
Hongkun Yu committed
948
949
    if self.use_proj:
      self.proj_layer = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
950
          units=self.hidden_size,
Hongkun Yu's avatar
Hongkun Yu committed
951
952
          kernel_initializer=self.initializer,
          activation=gelu,
Allen Wang's avatar
Allen Wang committed
953
          name="lm_projection/dense")
Hongkun Yu's avatar
Hongkun Yu committed
954
      self.proj_layer_norm = tf.keras.layers.LayerNormalization(
Allen Wang's avatar
Allen Wang committed
955
          axis=-1, epsilon=1e-12, name="lm_projection/LayerNorm")
Hongkun Yu's avatar
Hongkun Yu committed
956
    if not self.tie_weight:
Hongkun Yu's avatar
Hongkun Yu committed
957
      self.softmax_w = self.add_weight(
Allen Wang's avatar
Allen Wang committed
958
959
          "weight",
          shape=[self.vocab_size, self.hidden_size],
Hongkun Yu's avatar
Hongkun Yu committed
960
          initializer=self.initializer)
Hongkun Yu's avatar
Hongkun Yu committed
961

Hongkun Yu's avatar
Hongkun Yu committed
962
    self.softmax_b = self.add_weight(
Allen Wang's avatar
Allen Wang committed
963
        "bias", shape=[self.vocab_size], initializer=tf.zeros_initializer())
Hongkun Yu's avatar
Hongkun Yu committed
964
965
966

    super(LMLossLayer, self).build(unused_input_shapes)

Hongkun Yu's avatar
Hongkun Yu committed
967
  def call(self, hidden, target, lookup_table, target_mask):
Hongkun Yu's avatar
Hongkun Yu committed
968
    """Implements call() for the layer."""
Hongkun Yu's avatar
Hongkun Yu committed
969
970
    if self.use_proj:
      hidden = self.proj_layer_norm(self.proj_layer(hidden))
Hongkun Yu's avatar
Hongkun Yu committed
971
    if self.tie_weight:
Allen Wang's avatar
Allen Wang committed
972
      logits = tf.einsum("ibd,nd->ibn", hidden, lookup_table) + self.softmax_b
Hongkun Yu's avatar
Hongkun Yu committed
973
    else:
Allen Wang's avatar
Allen Wang committed
974
      logits = tf.einsum("ibd,nd->ibn", hidden, self.softmax_w) + self.softmax_b
Hongkun Yu's avatar
Hongkun Yu committed
975

Allen Wang's avatar
Allen Wang committed
976
977
    if self.use_one_hot:
      one_hot_target = tf.one_hot(target, self.vocab_size, dtype=logits.dtype)
Hongkun Yu's avatar
Hongkun Yu committed
978
979
      loss = -tf.reduce_sum(tf.nn.log_softmax(logits) * one_hot_target, -1)
    else:
Hongkun Yu's avatar
Hongkun Yu committed
980
981
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
          labels=target, logits=logits)
Hongkun Yu's avatar
Hongkun Yu committed
982

Hongkun Yu's avatar
Hongkun Yu committed
983
    total_loss = tf.reduce_sum(loss * target_mask) / tf.reduce_sum(target_mask)
Hongkun Yu's avatar
Hongkun Yu committed
984

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
985
    return total_loss, logits
Hongkun Yu's avatar
Hongkun Yu committed
986
987
988
989
990
991


class Summarization(tf.keras.layers.Layer):
  """The layer to pool the output from XLNet model into a vector."""

  def __init__(self,
Allen Wang's avatar
Allen Wang committed
992
993
994
995
996
               hidden_size,
               num_attention_heads,
               head_size,
               dropout_rate,
               attention_dropout_rate,
Hongkun Yu's avatar
Hongkun Yu committed
997
998
               initializer,
               use_proj=True,
Allen Wang's avatar
Allen Wang committed
999
               summary_type="last",
Hongkun Yu's avatar
Hongkun Yu committed
1000
1001
1002
1003
               **kwargs):
    """Constructs Summarization layer.

    Args:
Allen Wang's avatar
Allen Wang committed
1004
1005
1006
1007
1008
      hidden_size: int, the dimension of model hidden state.
      num_attention_heads: int, the number of attention heads.
      head_size: int, the dimension size of each attention head.
      dropout_rate: float, dropout rate.
      attention_dropout_rate: float, dropout rate on attention probabilities.
Hongkun Yu's avatar
Hongkun Yu committed
1009
1010
1011
1012
1013
1014
      initializer: Initializer used for parameters.
      use_proj: bool, whether to use projection layer for summarization.
      summary_type: Method used to summarize a sequence into a compact vector.
      **kwargs: Other parameters.
    """
    super(Summarization, self).__init__(**kwargs)
Allen Wang's avatar
Allen Wang committed
1015
1016
1017
    self.hidden_size = hidden_size
    self.num_attention_heads = num_attention_heads
    self.head_size = head_size
Hongkun Yu's avatar
Hongkun Yu committed
1018
1019
    self.initializer = initializer

Allen Wang's avatar
Allen Wang committed
1020
1021
    self.dropout_rate = dropout_rate
    self.attention_dropout_rate = attention_dropout_rate
Hongkun Yu's avatar
Hongkun Yu committed
1022
1023
1024
1025
1026
1027
1028
    self.use_proj = use_proj
    self.summary_type = summary_type

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
    if self.use_proj:
      self.proj_layer = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
1029
          units=self.hidden_size,
Hongkun Yu's avatar
Hongkun Yu committed
1030
1031
          kernel_initializer=self.initializer,
          activation=tf.nn.tanh,
Allen Wang's avatar
Allen Wang committed
1032
1033
          name="summary")
    self.dropout_layer = tf.keras.layers.Dropout(rate=self.dropout_rate)
Hongkun Yu's avatar
Hongkun Yu committed
1034
1035
1036
1037
1038

    super(Summarization, self).build(unused_input_shapes)

  def call(self, inputs):
    """Implements call() for the layer."""
Allen Wang's avatar
Allen Wang committed
1039
1040
1041
1042
    if self.summary_type == "last":
      summary = inputs[:, -1, :]
    elif self.summary_type == "first":
      summary = inputs[:, 0, :]
Hongkun Yu's avatar
Hongkun Yu committed
1043
    else:
Allen Wang's avatar
Allen Wang committed
1044
      raise ValueError("Invalid summary type provided: %s" % self.summary_type)
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
1045
1046
    if self.use_proj:
      summary = self.proj_layer(summary)
Hongkun Yu's avatar
Hongkun Yu committed
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
    summary = self.dropout_layer(summary)
    return summary


class ClassificationLossLayer(tf.keras.layers.Layer):
  """Layer computing cross entropy loss for classification task."""

  def __init__(self, n_class, initializer, **kwargs):
    """Constructs Summarization layer.

    Args:
      n_class: Number of tokens in vocabulary.
      initializer: Initializer used for parameters.
      **kwargs: Other parameters.
    """
    super(ClassificationLossLayer, self).__init__(**kwargs)

    self.n_class = n_class
    self.initializer = initializer

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
    self.proj_layer = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
1070
        units=self.n_class, kernel_initializer=self.initializer, name="logit")
Hongkun Yu's avatar
Hongkun Yu committed
1071
1072
1073

    super(ClassificationLossLayer, self).build(unused_input_shapes)

Hongkun Yu's avatar
Hongkun Yu committed
1074
  def call(self, hidden, labels):
Hongkun Yu's avatar
Hongkun Yu committed
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
    """Implements call() for the layer."""

    logits = self.proj_layer(hidden)
    one_hot_target = tf.one_hot(labels, self.n_class, dtype=hidden.dtype)  # pytype: disable=attribute-error
    loss = -tf.reduce_sum(tf.nn.log_softmax(logits) * one_hot_target, -1)

    return loss, logits


class QAXLNetModel(tf.keras.Model):
  """XLNet keras model combined with question answering loss layer.

  See the original paper: https://arxiv.org/pdf/1906.08237.pdf

  """

  def __init__(self, xlnet_config, run_config, start_n_top, end_n_top,
Allen Wang's avatar
Allen Wang committed
1092
               use_legacy_mask=True, **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
1093
    super(QAXLNetModel, self).__init__(**kwargs)
Allen Wang's avatar
Allen Wang committed
1094
1095
1096
    warnings.warn(
        "`QAXLNetModel` is deprecated, please use `XLNetSpanLabeler` instead.",
        DeprecationWarning, stacklevel=2)
Hongkun Yu's avatar
Hongkun Yu committed
1097
1098
1099
    self.run_config = run_config
    self.initializer = _get_initializer(run_config)
    self.xlnet_config = copy.deepcopy(xlnet_config)
Allen Wang's avatar
Allen Wang committed
1100
    self._use_legacy_mask = use_legacy_mask
Hongkun Yu's avatar
Hongkun Yu committed
1101

Allen Wang's avatar
Allen Wang committed
1102
1103
    self.xlnet_model = networks.XLNetBase(
        vocab_size=self.xlnet_config.n_token,
Hongkun Yu's avatar
Hongkun Yu committed
1104
        initializer=self.initializer,
Allen Wang's avatar
Allen Wang committed
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
        attention_type="bi",
        num_layers=self.xlnet_config.n_layer,
        hidden_size=self.xlnet_config.d_model,
        num_attention_heads=self.xlnet_config.n_head,
        head_size=self.xlnet_config.d_head,
        inner_size=self.xlnet_config.d_inner,
        tie_attention_biases=not self.xlnet_config.untie_r,
        inner_activation=self.xlnet_config.ff_activation,
        dropout_rate=self.run_config.dropout,
        attention_dropout_rate=self.run_config.dropout_att,
        two_stream=False,
        memory_length=self.run_config.mem_len,
        reuse_length=self.run_config.reuse_len,
Hongkun Yu's avatar
Hongkun Yu committed
1118
        bi_data=self.run_config.bi_data,
Allen Wang's avatar
Allen Wang committed
1119
1120
1121
        clamp_length=self.run_config.clamp_len,
        use_cls_mask=False,
        name="xlnet_model")
Hongkun Yu's avatar
Hongkun Yu committed
1122
1123

    self.qa_loss_layer = QALossLayer(
Allen Wang's avatar
Allen Wang committed
1124
        hidden_size=self.xlnet_config.d_model,
Hongkun Yu's avatar
Hongkun Yu committed
1125
1126
1127
        start_n_top=start_n_top,
        end_n_top=end_n_top,
        initializer=self.initializer,
Allen Wang's avatar
Allen Wang committed
1128
1129
        dropout_rate=self.run_config.dropout,
        name="qa_loss_layer")
Hongkun Yu's avatar
Hongkun Yu committed
1130
1131
1132
1133

  def call(self, features, training=False):
    """Implements call() for the layer."""

Allen Wang's avatar
Allen Wang committed
1134
1135
    input_ids = features["input_ids"]
    segment_ids = features["segment_ids"]
Allen Wang's avatar
Allen Wang committed
1136
    if self._use_legacy_mask:
1137
1138
      # Legacy input mask assumes `real` values are 0 and `padding`
      # values are 1.
Allen Wang's avatar
Allen Wang committed
1139
1140
1141
      input_mask = 1 - features["input_mask"]
    else:
      input_mask = features["input_mask"]
Hongkun Yu's avatar
Hongkun Yu committed
1142

Allen Wang's avatar
Allen Wang committed
1143
1144
    cls_index = tf.reshape(features["cls_index"], [-1])
    p_mask = features["p_mask"]
Hongkun Yu's avatar
Hongkun Yu committed
1145

Allen Wang's avatar
Allen Wang committed
1146
1147
    attention_output, new_mems = (
        self.xlnet_model(input_ids, segment_ids, input_mask))
Hongkun Yu's avatar
Hongkun Yu committed
1148
1149
1150

    if training:
      loss, logits = self.qa_loss_layer(
Allen Wang's avatar
Allen Wang committed
1151
          hidden=attention_output,
Hongkun Yu's avatar
Hongkun Yu committed
1152
1153
          p_mask=p_mask,
          cls_index=cls_index,
Allen Wang's avatar
Allen Wang committed
1154
1155
1156
          start_positions=features["start_positions"],
          end_positions=features["end_positions"],
          is_impossible=features["is_impossible"])
Hongkun Yu's avatar
Hongkun Yu committed
1157
      self.add_loss(loss)
Hongkun Yu's avatar
Hongkun Yu committed
1158
      return new_mems, logits
Hongkun Yu's avatar
Hongkun Yu committed
1159
1160
    else:
      results = self.qa_loss_layer(
Allen Wang's avatar
Allen Wang committed
1161
          hidden=attention_output, p_mask=p_mask, cls_index=cls_index)
Hongkun Yu's avatar
Hongkun Yu committed
1162
1163
1164
1165
      return results


class QALossLayer(tf.keras.layers.Layer):
Hongkun Yu's avatar
Hongkun Yu committed
1166
  """Layer computing position and regression loss for question answering task."""
Hongkun Yu's avatar
Hongkun Yu committed
1167

Allen Wang's avatar
Allen Wang committed
1168
1169
  def __init__(self, hidden_size, start_n_top, end_n_top, initializer,
               dropout_rate, **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
1170
1171
1172
    """Constructs Summarization layer.

    Args:
Allen Wang's avatar
Allen Wang committed
1173
      hidden_size: Int, the hidden size.
Hongkun Yu's avatar
Hongkun Yu committed
1174
1175
1176
      start_n_top: Beam size for span start.
      end_n_top: Beam size for span end.
      initializer: Initializer used for parameters.
Allen Wang's avatar
Allen Wang committed
1177
      dropout_rate: float, dropout rate.
Hongkun Yu's avatar
Hongkun Yu committed
1178
1179
1180
      **kwargs: Other parameters.
    """
    super(QALossLayer, self).__init__(**kwargs)
Allen Wang's avatar
Allen Wang committed
1181
    self.hidden_size = hidden_size
Hongkun Yu's avatar
Hongkun Yu committed
1182
1183
1184
    self.start_n_top = start_n_top
    self.end_n_top = end_n_top
    self.initializer = initializer
Allen Wang's avatar
Allen Wang committed
1185
    self.dropout_rate = dropout_rate
Hongkun Yu's avatar
Hongkun Yu committed
1186
1187
1188
1189

  def build(self, unused_input_shapes):
    """Implements build() for the layer."""
    self.start_logits_proj_layer = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
1190
        units=1, kernel_initializer=self.initializer, name="start_logits/dense")
Hongkun Yu's avatar
Hongkun Yu committed
1191
    self.end_logits_proj_layer0 = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
1192
        units=self.hidden_size,
Hongkun Yu's avatar
Hongkun Yu committed
1193
1194
        kernel_initializer=self.initializer,
        activation=tf.nn.tanh,
Allen Wang's avatar
Allen Wang committed
1195
        name="end_logits/dense_0")
Hongkun Yu's avatar
Hongkun Yu committed
1196
    self.end_logits_proj_layer1 = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
1197
        units=1, kernel_initializer=self.initializer, name="end_logits/dense_1")
Hongkun Yu's avatar
Hongkun Yu committed
1198
    self.end_logits_layer_norm = tf.keras.layers.LayerNormalization(
Allen Wang's avatar
Allen Wang committed
1199
        axis=-1, epsilon=1e-12, name="end_logits/LayerNorm")
Hongkun Yu's avatar
Hongkun Yu committed
1200
    self.answer_class_proj_layer0 = tf.keras.layers.Dense(
Allen Wang's avatar
Allen Wang committed
1201
        units=self.hidden_size,
Hongkun Yu's avatar
Hongkun Yu committed
1202
1203
        kernel_initializer=self.initializer,
        activation=tf.nn.tanh,
Allen Wang's avatar
Allen Wang committed
1204
        name="answer_class/dense_0")
Hongkun Yu's avatar
Hongkun Yu committed
1205
1206
1207
1208
    self.answer_class_proj_layer1 = tf.keras.layers.Dense(
        units=1,
        kernel_initializer=self.initializer,
        use_bias=False,
Allen Wang's avatar
Allen Wang committed
1209
1210
        name="answer_class/dense_1")
    self.ans_feature_dropout = tf.keras.layers.Dropout(rate=self.dropout_rate)
Hongkun Yu's avatar
Hongkun Yu committed
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
    super(QALossLayer, self).build(unused_input_shapes)

  def __call__(self, hidden, p_mask, cls_index, **kwargs):
    return super(QALossLayer, self).__call__(
        (hidden, p_mask, cls_index, kwargs))

  def call(self, inputs, training=False):
    """Implements call() for the layer."""
    hidden, p_mask, cls_index, kwargs = inputs
    return_dict = {}
Allen Wang's avatar
Allen Wang committed
1221
    seq_len = tf.shape(hidden)[1]
Hongkun Yu's avatar
Hongkun Yu committed
1222

Allen Wang's avatar
Allen Wang committed
1223
    hidden = tf.transpose(hidden, [1, 0, 2])
Hongkun Yu's avatar
Hongkun Yu committed
1224
1225
1226
1227
1228
    start_logits = self.start_logits_proj_layer(hidden)
    start_logits = tf.transpose(tf.squeeze(start_logits, -1), [1, 0])
    start_logits_masked = start_logits * (1 - p_mask) - 1e30 * p_mask
    start_log_probs = tf.nn.log_softmax(start_logits_masked, -1)
    if training:
Allen Wang's avatar
Allen Wang committed
1229
1230
1231
      start_positions = kwargs["start_positions"]
      end_positions = kwargs["end_positions"]
      is_impossible = kwargs["is_impossible"]
Hongkun Yu's avatar
Hongkun Yu committed
1232
1233
1234
      start_positions = tf.reshape(start_positions, [-1])
      start_index = tf.one_hot(
          start_positions, depth=seq_len, axis=-1, dtype=tf.float32)
Allen Wang's avatar
Allen Wang committed
1235
      start_features = tf.einsum("lbh,bl->bh", hidden, start_index)
Hongkun Yu's avatar
Hongkun Yu committed
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
      start_features = tf.tile(start_features[None], [seq_len, 1, 1])
      end_logits = self.end_logits_proj_layer0(
          tf.concat([hidden, start_features], axis=-1))

      end_logits = self.end_logits_layer_norm(end_logits)

      end_logits = self.end_logits_proj_layer1(end_logits)
      end_logits = tf.transpose(tf.squeeze(end_logits, -1), [1, 0])
      end_logits_masked = end_logits * (1 - p_mask) - 1e30 * p_mask
      end_log_probs = tf.nn.log_softmax(end_logits_masked, -1)
    else:
      # during inference, compute the end logits based on beam search

      start_top_log_probs, start_top_index = tf.nn.top_k(
          start_log_probs, k=self.start_n_top)
      start_index = tf.one_hot(
          start_top_index, depth=seq_len, axis=-1, dtype=tf.float32)
Allen Wang's avatar
Allen Wang committed
1253
      start_features = tf.einsum("lbh,bkl->bkh", hidden, start_index)
Hongkun Yu's avatar
Hongkun Yu committed
1254
1255
1256
1257
      end_input = tf.tile(hidden[:, :, None], [1, 1, self.start_n_top, 1])
      start_features = tf.tile(start_features[None], [seq_len, 1, 1, 1])
      end_input = tf.concat([end_input, start_features], axis=-1)
      end_logits = self.end_logits_proj_layer0(end_input)
Allen Wang's avatar
Allen Wang committed
1258
      end_logits = tf.reshape(end_logits, [seq_len, -1, self.hidden_size])
Hongkun Yu's avatar
Hongkun Yu committed
1259
1260
1261
      end_logits = self.end_logits_layer_norm(end_logits)

      end_logits = tf.reshape(end_logits,
Allen Wang's avatar
Allen Wang committed
1262
                              [seq_len, -1, self.start_n_top, self.hidden_size])
Hongkun Yu's avatar
Hongkun Yu committed
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277

      end_logits = self.end_logits_proj_layer1(end_logits)
      end_logits = tf.reshape(end_logits, [seq_len, -1, self.start_n_top])
      end_logits = tf.transpose(end_logits, [1, 2, 0])
      end_logits_masked = end_logits * (
          1 - p_mask[:, None]) - 1e30 * p_mask[:, None]
      end_log_probs = tf.nn.log_softmax(end_logits_masked, -1)
      end_top_log_probs, end_top_index = tf.nn.top_k(
          end_log_probs, k=self.end_n_top)
      end_top_log_probs = tf.reshape(end_top_log_probs,
                                     [-1, self.start_n_top * self.end_n_top])
      end_top_index = tf.reshape(end_top_index,
                                 [-1, self.start_n_top * self.end_n_top])

    if training:
Allen Wang's avatar
Allen Wang committed
1278
1279
      return_dict["start_log_probs"] = start_log_probs
      return_dict["end_log_probs"] = end_log_probs
Hongkun Yu's avatar
Hongkun Yu committed
1280
    else:
Allen Wang's avatar
Allen Wang committed
1281
1282
1283
1284
      return_dict["start_top_log_probs"] = start_top_log_probs
      return_dict["start_top_index"] = start_top_index
      return_dict["end_top_log_probs"] = end_top_log_probs
      return_dict["end_top_index"] = end_top_index
Hongkun Yu's avatar
Hongkun Yu committed
1285
1286
1287
1288
    # an additional layer to predict answerability

    # get the representation of CLS
    cls_index = tf.one_hot(cls_index, seq_len, axis=-1, dtype=tf.float32)
Allen Wang's avatar
Allen Wang committed
1289
    cls_feature = tf.einsum("lbh,bl->bh", hidden, cls_index)
Hongkun Yu's avatar
Hongkun Yu committed
1290
1291

    # get the representation of START
Allen Wang's avatar
Allen Wang committed
1292
1293
    start_p = tf.nn.softmax(start_logits_masked, axis=-1, name="softmax_start")
    start_feature = tf.einsum("lbh,bl->bh", hidden, start_p)
Hongkun Yu's avatar
Hongkun Yu committed
1294
1295
1296
1297
1298
1299

    ans_feature = tf.concat([start_feature, cls_feature], -1)
    ans_feature = self.answer_class_proj_layer0(ans_feature)
    ans_feature = self.ans_feature_dropout(ans_feature)
    cls_logits = self.answer_class_proj_layer1(ans_feature)
    cls_logits = tf.squeeze(cls_logits, -1)
Allen Wang's avatar
Allen Wang committed
1300
    return_dict["cls_logits"] = cls_logits
Hongkun Yu's avatar
Hongkun Yu committed
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323

    if not training:
      return return_dict

    def compute_loss(log_probs, positions):
      one_hot_positions = tf.one_hot(positions, depth=seq_len, dtype=tf.float32)

      loss = -tf.reduce_sum(one_hot_positions * log_probs, axis=-1)
      loss = tf.reduce_mean(loss)
      return loss

    start_loss = compute_loss(start_log_probs, start_positions)
    end_loss = compute_loss(end_log_probs, end_positions)

    total_loss = (start_loss + end_loss) * 0.5

    is_impossible = tf.reshape(is_impossible, [-1])
    regression_loss = tf.nn.sigmoid_cross_entropy_with_logits(
        labels=is_impossible, logits=cls_logits)
    regression_loss = tf.reduce_mean(regression_loss)

    total_loss += regression_loss * 0.5
    return total_loss, cls_logits