bert_dataset.py 8.41 KB
Newer Older
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
1
# coding=utf-8
Mohammad's avatar
Mohammad committed
2
# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
16
"""BERT Style dataset."""
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
17

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
18
import os
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
19
20
21
22
23
24
import time

import numpy as np
import torch
from torch.utils.data import Dataset

Neel Kant's avatar
Neel Kant committed
25
from megatron import get_tokenizer, get_args
26
from megatron import mpu
27
from megatron.data.indexed_dataset import make_dataset as make_indexed_dataset
28
from megatron import print_rank_0
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
29

Neel Kant's avatar
Neel Kant committed
30

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
31
class BertDataset(Dataset):
32

33
    def __init__(self, name, indexed_dataset, data_prefix,
34
35
                 num_epochs, max_num_samples, masked_lm_prob,
                 max_seq_length, short_seq_prob, seed):
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
36
37

        # Params to store.
38
        self.name = name
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
39
40
41
42
        self.seed = seed
        self.masked_lm_prob = masked_lm_prob
        self.max_seq_length = max_seq_length

43
        # Dataset.
44
45
        self.indexed_dataset = indexed_dataset

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
46
        # Build the samples mapping.
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
47
48
49
50
51
52
        self.samples_mapping = get_samples_mapping_(self.indexed_dataset,
                                                    data_prefix,
                                                    num_epochs,
                                                    max_num_samples,
                                                    self.max_seq_length,
                                                    short_seq_prob,
53
54
                                                    self.seed,
                                                    self.name)
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
55
56

        # Vocab stuff.
57
58
59
60
61
62
63
        tokenizer = get_tokenizer()
        self.vocab_id_list = list(tokenizer.inv_vocab.keys())
        self.vocab_id_to_token_dict = tokenizer.inv_vocab
        self.cls_id = tokenizer.cls
        self.sep_id = tokenizer.sep
        self.mask_id = tokenizer.mask
        self.pad_id = tokenizer.pad
Neel Kant's avatar
Neel Kant committed
64
65
        from megatron.data.dataset_utils import build_training_sample
        self.build_sample_fn = build_training_sample
66

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
67
    def __len__(self):
68
        return self.samples_mapping.shape[0]
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
69
70

    def __getitem__(self, idx):
71
72
        start_idx, end_idx, seq_length = self.samples_mapping[idx]
        sample = [self.indexed_dataset[i] for i in range(start_idx, end_idx)]
73
74
75
        # Note that this rng state should be numpy and not python since
        # python randint is inclusive whereas the numpy one is exclusive.
        np_rng = np.random.RandomState(seed=(self.seed + idx))
Neel Kant's avatar
Neel Kant committed
76
77
78
79
80
81
82
        return self.build_sample_fn(sample, seq_length,
                                    self.max_seq_length,  # needed for padding
                                    self.vocab_id_list,
                                    self.vocab_id_to_token_dict,
                                    self.cls_id, self.sep_id,
                                    self.mask_id, self.pad_id,
                                    self.masked_lm_prob, np_rng)
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
83

84

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
85
def get_indexed_dataset_(data_prefix, data_impl, skip_warmup):
86
87
88

    print_rank_0(' > building dataset index ...')

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
89
90
91
92
    start_time = time.time()
    indexed_dataset = make_indexed_dataset(data_prefix,
                                           data_impl,
                                           skip_warmup)
93
94
95
96
97
98
99
100
101
102
    assert indexed_dataset.sizes.shape[0] == indexed_dataset.doc_idx[-1]
    print_rank_0(' > finished creating indexed dataset in {:4f} '
                 'seconds'.format(time.time() - start_time))

    print_rank_0(' > indexed dataset stats:')
    print_rank_0('    number of documents: {}'.format(
        indexed_dataset.doc_idx.shape[0] - 1))
    print_rank_0('    number of sentences: {}'.format(
        indexed_dataset.sizes.shape[0]))

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
103
104
105
    return indexed_dataset


106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def get_train_valid_test_split_(splits_string, size):
    """ Get dataset splits from comma or '/' separated string list."""

    splits = []
    if splits_string.find(',') != -1:
        splits = [float(s) for s in splits_string.split(',')]
    elif splits_string.find('/') != -1:
        splits = [float(s) for s in splits_string.split('/')]
    else:
        splits = [float(splits_string)]
    while len(splits) < 3:
        splits.append(0.)
    splits = splits[:3]
    splits_sum = sum(splits)
    assert splits_sum > 0.0
Neel Kant's avatar
Neel Kant committed
121
    splits = [split / splits_sum for split in splits]
122
123
124
125
126
127
128
129
130
131
132
133
    splits_index = [0]
    for index, split in enumerate(splits):
        splits_index.append(splits_index[index] +
                            int(round(split * float(size))))
    diff = splits_index[-1] - size
    for index in range(1, len(splits_index)):
        splits_index[index] -= diff
    assert len(splits_index) == 4
    assert splits_index[-1] == size
    return splits_index


Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
134
135
136
137
138
139
def get_samples_mapping_(indexed_dataset,
                         data_prefix,
                         num_epochs,
                         max_num_samples,
                         max_seq_length,
                         short_seq_prob,
140
141
                         seed,
                         name):
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
142
    if not num_epochs:
143
        if not max_num_samples:
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
144
145
146
147
148
149
150
151
            raise ValueError("Need to specify either max_num_samples "
                             "or num_epochs")
        num_epochs = np.iinfo(np.int32).max - 1
    if not max_num_samples:
        max_num_samples = np.iinfo(np.int64).max - 1

    # Filename of the index mapping
    indexmap_filename = data_prefix
152
153
154
155
156
    indexmap_filename += '_{}_indexmap'.format(name)
    if num_epochs != (np.iinfo(np.int32).max - 1):
        indexmap_filename += '_{}ep'.format(num_epochs)
    if max_num_samples != (np.iinfo(np.int64).max - 1):
        indexmap_filename += '_{}mns'.format(max_num_samples)
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
157
158
159
160
161
162
163
164
    indexmap_filename += '_{}msl'.format(max_seq_length)
    indexmap_filename += '_{:0.2f}ssp'.format(short_seq_prob)
    indexmap_filename += '_{}s'.format(seed)
    indexmap_filename += '.npy'

    # Build the indexed mapping if not exist.
    if torch.distributed.get_rank() == 0 and \
       not os.path.isfile(indexmap_filename):
165
        print(' > WARNING: could not find index map file {}, building '
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
166
              'the indices on rank 0 ...'.format(indexmap_filename))
167

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
168
169
170
171
172
173
        # Make sure the types match the helpers input types.
        assert indexed_dataset.doc_idx.dtype == np.int64
        assert indexed_dataset.sizes.dtype == np.int32

        # Build samples mapping
        verbose = torch.distributed.get_rank() == 0
174
        start_time = time.time()
175
176
        print_rank_0(' > building sapmles index mapping for {} ...'.format(
            name))
177
178
179
        # First compile and then import.
        from megatron.data.dataset_utils import compile_helper
        compile_helper()
Mohammad's avatar
Mohammad committed
180
        from megatron.data import helpers
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
181
182
183
184
185
        samples_mapping = helpers.build_mapping(
            indexed_dataset.doc_idx,
            indexed_dataset.sizes,
            num_epochs,
            max_num_samples,
Neel Kant's avatar
Neel Kant committed
186
            max_seq_length - 3,  # account for added tokens
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
187
188
189
            short_seq_prob,
            seed,
            verbose)
190
        print_rank_0(' > done building sapmles index maping')
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
191
        np.save(indexmap_filename, samples_mapping, allow_pickle=True)
192
193
        print_rank_0(' > saved the index mapping in {}'.format(
            indexmap_filename))
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
194
        # Make sure all the ranks have built the mapping
195
        print_rank_0(' > elasped time to build and save samples mapping '
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
196
197
                     '(seconds): {:4f}'.format(
                         time.time() - start_time))
198
199
200
201
202
203
204
    # This should be a barrier but nccl barrier assumes
    # device_index=rank which is not the case for model
    # parallel case
    counts = torch.cuda.LongTensor([1])
    torch.distributed.all_reduce(counts, group=mpu.get_data_parallel_group())
    assert counts[0].item() == torch.distributed.get_world_size(
        group=mpu.get_data_parallel_group())
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
205
206

    # Load indexed dataset.
207
    print_rank_0(' > loading indexed mapping from {}'.format(
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
208
209
210
        indexmap_filename))
    start_time = time.time()
    samples_mapping = np.load(indexmap_filename, allow_pickle=True)
211
    print_rank_0('    loaded indexed file in {:3.3f} seconds'.format(
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
212
        time.time() - start_time))
213
    print_rank_0('    total number of samples: {}'.format(
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
214
        samples_mapping.shape[0]))
215

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
216
    return samples_mapping