distillation_test.py 7.07 KB
Newer Older
Chen Chen's avatar
Chen Chen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for official.nlp.projects.mobilebert.distillation."""
import os

from absl import logging
Chen Chen's avatar
Chen Chen committed
19
from absl.testing import parameterized
Chen Chen's avatar
Chen Chen committed
20
import tensorflow as tf
Chen Chen's avatar
Chen Chen committed
21

Chen Chen's avatar
Chen Chen committed
22
23
24
from official.core import config_definitions as cfg
from official.modeling import optimization
from official.modeling import tf_utils
Le Hou's avatar
Le Hou committed
25
from official.modeling.fast_training.progressive import trainer as prog_trainer_lib
Chen Chen's avatar
Chen Chen committed
26
27
28
29
30
from official.nlp.configs import bert
from official.nlp.configs import encoders
from official.nlp.data import pretrain_dataloader
from official.nlp.modeling import layers
from official.nlp.modeling import models
31
from official.projects.mobilebert import distillation
Chen Chen's avatar
Chen Chen committed
32
33


Chen Chen's avatar
Chen Chen committed
34
class DistillationTest(tf.test.TestCase, parameterized.TestCase):
Chen Chen's avatar
Chen Chen committed
35

Chen Chen's avatar
Chen Chen committed
36
37
  def prepare_config(self, teacher_block_num, student_block_num,
                     transfer_teacher_layers):
Chen Chen's avatar
Chen Chen committed
38
    # using small model for testing
Chen Chen's avatar
Chen Chen committed
39
    task_config = distillation.BertDistillationTaskConfig(
Chen Chen's avatar
Chen Chen committed
40
41
42
43
        teacher_model=bert.PretrainerConfig(
            encoder=encoders.EncoderConfig(
                type='mobilebert',
                mobilebert=encoders.MobileBertEncoderConfig(
Chen Chen's avatar
Chen Chen committed
44
                    num_blocks=teacher_block_num)),
Chen Chen's avatar
Chen Chen committed
45
46
47
48
49
50
51
52
53
54
55
56
            cls_heads=[
                bert.ClsHeadConfig(
                    inner_dim=256,
                    num_classes=2,
                    dropout_rate=0.1,
                    name='next_sentence')
            ],
            mlm_activation='gelu'),
        student_model=bert.PretrainerConfig(
            encoder=encoders.EncoderConfig(
                type='mobilebert',
                mobilebert=encoders.MobileBertEncoderConfig(
Chen Chen's avatar
Chen Chen committed
57
                    num_blocks=student_block_num)),
Chen Chen's avatar
Chen Chen committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
            cls_heads=[
                bert.ClsHeadConfig(
                    inner_dim=256,
                    num_classes=2,
                    dropout_rate=0.1,
                    name='next_sentence')
            ],
            mlm_activation='relu'),
        train_data=pretrain_dataloader.BertPretrainDataConfig(
            input_path='dummy',
            max_predictions_per_seq=76,
            seq_length=512,
            global_batch_size=10),
        validation_data=pretrain_dataloader.BertPretrainDataConfig(
            input_path='dummy',
            max_predictions_per_seq=76,
            seq_length=512,
            global_batch_size=10))

    # set only 1 step for each stage
    progressive_config = distillation.BertDistillationProgressiveConfig()
Chen Chen's avatar
Chen Chen committed
79
80
    progressive_config.layer_wise_distill_config.transfer_teacher_layers = (
        transfer_teacher_layers)
Chen Chen's avatar
Chen Chen committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    progressive_config.layer_wise_distill_config.num_steps = 1
    progressive_config.pretrain_distill_config.num_steps = 1

    optimization_config = optimization.OptimizationConfig(
        optimizer=optimization.OptimizerConfig(
            type='lamb',
            lamb=optimization.LAMBConfig(
                weight_decay_rate=0.0001,
                exclude_from_weight_decay=[
                    'LayerNorm', 'layer_norm', 'bias', 'no_norm'
                ])),
        learning_rate=optimization.LrConfig(
            type='polynomial',
            polynomial=optimization.PolynomialLrConfig(
                initial_learning_rate=1.5e-3,
                decay_steps=10000,
                end_learning_rate=1.5e-3)),
        warmup=optimization.WarmupConfig(
            type='linear',
            linear=optimization.LinearWarmupConfig(warmup_learning_rate=0)))

Chen Chen's avatar
Chen Chen committed
102
103
    exp_config = cfg.ExperimentConfig(
        task=task_config,
Chen Chen's avatar
Chen Chen committed
104
105
106
107
108
        trainer=prog_trainer_lib.ProgressiveTrainerConfig(
            progressive=progressive_config,
            optimizer_config=optimization_config))

    # Create a teacher model checkpoint.
Chen Chen's avatar
Chen Chen committed
109
110
    teacher_encoder = encoders.build_encoder(task_config.teacher_model.encoder)
    pretrainer_config = task_config.teacher_model
Chen Chen's avatar
Chen Chen committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    if pretrainer_config.cls_heads:
      teacher_cls_heads = [
          layers.ClassificationHead(**cfg.as_dict())
          for cfg in pretrainer_config.cls_heads
      ]
    else:
      teacher_cls_heads = []

    masked_lm = layers.MobileBertMaskedLM(
        embedding_table=teacher_encoder.get_embedding_table(),
        activation=tf_utils.get_activation(pretrainer_config.mlm_activation),
        initializer=tf.keras.initializers.TruncatedNormal(
            stddev=pretrainer_config.mlm_initializer_range),
        name='cls/predictions')
    teacher_pretrainer = models.BertPretrainerV2(
        encoder_network=teacher_encoder,
        classification_heads=teacher_cls_heads,
        customized_masked_lm=masked_lm)

    # The model variables will be created after the forward call.
    _ = teacher_pretrainer(teacher_pretrainer.inputs)
    teacher_pretrainer_ckpt = tf.train.Checkpoint(
        **teacher_pretrainer.checkpoint_items)
    teacher_ckpt_path = os.path.join(self.get_temp_dir(), 'teacher_model.ckpt')
    teacher_pretrainer_ckpt.save(teacher_ckpt_path)
Chen Chen's avatar
Chen Chen committed
136
137
138
    exp_config.task.teacher_model_init_checkpoint = self.get_temp_dir()

    return exp_config
Chen Chen's avatar
Chen Chen committed
139

Chen Chen's avatar
Chen Chen committed
140
141
142
143
144
  @parameterized.parameters((2, 2, None), (4, 2, [1, 3]))
  def test_task(self, teacher_block_num, student_block_num,
                transfer_teacher_layers):
    exp_config = self.prepare_config(teacher_block_num, student_block_num,
                                     transfer_teacher_layers)
Chen Chen's avatar
Chen Chen committed
145
146
    bert_distillation_task = distillation.BertDistillationTask(
        strategy=tf.distribute.get_strategy(),
Chen Chen's avatar
Chen Chen committed
147
148
149
        progressive=exp_config.trainer.progressive,
        optimizer_config=exp_config.trainer.optimizer_config,
        task_config=exp_config.task)
Chen Chen's avatar
Chen Chen committed
150
151
152
153
154
155
156
157
158
    metrics = bert_distillation_task.build_metrics()
    train_dataset = bert_distillation_task.get_train_dataset(stage_id=0)
    train_iterator = iter(train_dataset)

    eval_dataset = bert_distillation_task.get_eval_dataset(stage_id=0)
    eval_iterator = iter(eval_dataset)
    optimizer = tf.keras.optimizers.SGD(lr=0.1)

    # test train/val step for all stages, including the last pretraining stage
Chen Chen's avatar
Chen Chen committed
159
    for stage in range(student_block_num + 1):
Chen Chen's avatar
Chen Chen committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
      step = stage
      bert_distillation_task.update_pt_stage(step)
      model = bert_distillation_task.get_model(stage, None)
      bert_distillation_task.initialize(model)
      bert_distillation_task.train_step(next(train_iterator), model, optimizer,
                                        metrics=metrics)
      bert_distillation_task.validation_step(next(eval_iterator), model,
                                             metrics=metrics)

    logging.info('begin to save and load model checkpoint')
    ckpt = tf.train.Checkpoint(model=model)
    ckpt.save(self.get_temp_dir())

if __name__ == '__main__':
  tf.test.main()