run_tf_glue.py 2.96 KB
Newer Older
1
import os
thomwolf's avatar
thomwolf committed
2
3
import tensorflow as tf
import tensorflow_datasets
thomwolf's avatar
thomwolf committed
4
from transformers import BertTokenizer, TFBertForSequenceClassification, glue_convert_examples_to_features, BertForSequenceClassification
thomwolf's avatar
thomwolf committed
5

6
7
8
# script parameters
BATCH_SIZE = 32
EVAL_BATCH_SIZE = BATCH_SIZE * 2
9
10
11
12
13
USE_XLA = False
USE_AMP = False

tf.config.optimizer.set_jit(USE_XLA)
tf.config.optimizer.set_experimental_options({"auto_mixed_precision": USE_AMP})
14
15

# Load tokenizer and model from pretrained model/vocabulary
thomwolf's avatar
thomwolf committed
16
tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
thomwolf's avatar
thomwolf committed
17
model = TFBertForSequenceClassification.from_pretrained('bert-base-cased')
18
19
20
21
22

# Load dataset via TensorFlow Datasets
data, info = tensorflow_datasets.load('glue/mrpc', with_info=True)
train_examples = info.splits['train'].num_examples
valid_examples = info.splits['validation'].num_examples
thomwolf's avatar
thomwolf committed
23

thomwolf's avatar
thomwolf committed
24
# Prepare dataset for GLUE as a tf.data.Dataset instance
thomwolf's avatar
thomwolf committed
25
26
train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, 128, 'mrpc')
valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, 128, 'mrpc')
27
28
train_dataset = train_dataset.shuffle(128).batch(BATCH_SIZE).repeat(-1)
valid_dataset = valid_dataset.batch(EVAL_BATCH_SIZE)
thomwolf's avatar
thomwolf committed
29

thomwolf's avatar
thomwolf committed
30
# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule 
31
32
33
34
opt = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08)
if USE_AMP:
    # loss scaling is currently required when using mixed precision
    opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic')
thomwolf's avatar
thomwolf committed
35
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
thomwolf's avatar
thomwolf committed
36
metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')
37
model.compile(optimizer=opt, loss=loss, metrics=[metric])
thomwolf's avatar
thomwolf committed
38
39

# Train and evaluate using tf.keras.Model.fit()
40
41
train_steps = train_examples//BATCH_SIZE
valid_steps = valid_examples//EVAL_BATCH_SIZE
thomwolf's avatar
thomwolf committed
42

43
44
45
46
47
history = model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps,
                    validation_data=valid_dataset, validation_steps=valid_steps)

# Save TF2 model
os.makedirs('./save/', exist_ok=True)
thomwolf's avatar
thomwolf committed
48
model.save_pretrained('./save/')
49
50

# Load the TensorFlow model in PyTorch for inspection
thomwolf's avatar
thomwolf committed
51
pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True)
thomwolf's avatar
thomwolf committed
52

thomwolf's avatar
thomwolf committed
53
# Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task
54
55
56
sentence_0 = 'This research was consistent with his findings.'
sentence_1 = 'His findings were compatible with this research.'
sentence_2 = 'His findings were not compatible with this research.'
thomwolf's avatar
thomwolf committed
57
58
59
60
61
inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')
inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt')

pred_1 = pytorch_model(**inputs_1)[0].argmax().item()
pred_2 = pytorch_model(**inputs_2)[0].argmax().item()
62
63
print('sentence_1 is', 'a paraphrase' if pred_1 else 'not a paraphrase', 'of sentence_0')
print('sentence_2 is', 'a paraphrase' if pred_2 else 'not a paraphrase', 'of sentence_0')