helpers.cpp 26.5 KB
Newer Older
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
1
2
/*
 coding=utf-8
Mohammad's avatar
Mohammad committed
3
 Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */


19
/* Helper methods for fast index mapping builds */
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
20
21
22
23
24

#include <algorithm>
#include <iostream>
#include <limits>
#include <math.h>
25
#include <stdexcept>
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
26
27
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
28
#include <random>
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
29
30
31
32

namespace py = pybind11;
using namespace std;

33
const int32_t LONG_SENTENCE_LEN = 512;
34

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
35

mohammad's avatar
mohammad committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
void build_blending_indices(py::array_t<uint8_t>& dataset_index,
			    py::array_t<int64_t>& dataset_sample_index,
			    const py::array_t<double>& weights,
			    const int32_t num_datasets,
			    const int64_t size, const bool verbose) {
  /* Given multiple datasets and a weighting array, build samples
   such that it follows those wieghts.*/

  if (verbose) {
    std::cout << "> building indices for blendable datasets ..." << std::endl;
  }

  // Get the pointer access without the checks.
  auto dataset_index_ptr = dataset_index.mutable_unchecked<1>();
  auto dataset_sample_index_ptr = dataset_sample_index.mutable_unchecked<1>();
  auto weights_ptr = weights.unchecked<1>();

  // Initialize buffer for number of samples used for each dataset.
  int64_t current_samples[num_datasets];
  for(int64_t i = 0; i < num_datasets; ++i) {
    current_samples[i] = 0;
  }

  // For each sample:
  for(int64_t sample_idx = 0; sample_idx < size; ++sample_idx) {

    // Determine where the max error in sampling is happening.
mohammad's avatar
mohammad committed
63
    auto sample_idx_double = std::max(static_cast<double>(sample_idx), 1.0);
mohammad's avatar
mohammad committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    int64_t max_error_index = 0;
    double max_error = weights_ptr[0] * sample_idx_double -
      static_cast<double>(current_samples[0]);
    for (int64_t dataset_idx = 1; dataset_idx < num_datasets; ++dataset_idx) {
      double error = weights_ptr[dataset_idx] * sample_idx_double -
	static_cast<double>(current_samples[dataset_idx]);
      if (error > max_error) {
	max_error = error;
	max_error_index = dataset_idx;
      }
    }

    // Populate the indices.
    dataset_index_ptr[sample_idx] = static_cast<uint8_t>(max_error_index);
    dataset_sample_index_ptr[sample_idx] = current_samples[max_error_index];

    // Update the total samples.
    current_samples[max_error_index] += 1;
    
  }

  // print info
  if (verbose) {
    std::cout << " > sample ratios:" << std::endl;
    for (int64_t dataset_idx = 0; dataset_idx < num_datasets; ++dataset_idx) {
mohammad's avatar
mohammad committed
89
      auto ratio = static_cast<double>(current_samples[dataset_idx]) /
mohammad's avatar
mohammad committed
90
91
92
93
94
95
96
97
98
	static_cast<double>(size);
      std::cout << "   dataset " << dataset_idx << ", input: " <<
	weights_ptr[dataset_idx] << ", achieved: " << ratio << std::endl; 
    }
  }

}


Mohammad's avatar
Mohammad committed
99
100
101
102
103
py::array build_sample_idx(const py::array_t<int32_t>& sizes_,
			   const py::array_t<int32_t>& doc_idx_,
			   const int32_t seq_length,
			   const int32_t num_epochs,
			   const int64_t tokens_per_epoch) {
Mohammad's avatar
Mohammad committed
104
105
106
107
    /* Sample index (sample_idx) is used for gpt2 like dataset for which
       the documents are flattened and the samples are built based on this
       1-D flatten array. It is a 2D array with sizes [number-of-samples + 1, 2]
       where [..., 0] contains the index into `doc_idx` and [..., 1] is the
Mohammad's avatar
Mohammad committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
       starting offset in that document.*/

    // Consistency checks.
    assert(seq_length > 1);
    assert(num_epochs > 0);
    assert(tokens_per_epoch > 1);

    // Remove bound checks.
    auto sizes = sizes_.unchecked<1>();
    auto doc_idx = doc_idx_.unchecked<1>();

    // Mapping and it's length (1D).
    int64_t num_samples = (num_epochs * tokens_per_epoch - 1) / seq_length;
    int32_t* sample_idx = new int32_t[2*(num_samples+1)];

    cout << "    using:" << endl << std::flush;
    cout << "     number of documents:       " <<
      doc_idx_.shape(0) / num_epochs << endl << std::flush;
    cout << "     number of epochs:          " << num_epochs <<
      endl << std::flush;
    cout << "     sequence length:           " << seq_length <<
      endl << std::flush;
    cout << "     total number of samples:   " << num_samples <<
      endl << std::flush;

    // Index into sample_idx.
    int64_t sample_index = 0;
    // Index into doc_idx.
    int64_t doc_idx_index = 0;
    // Begining offset for each document.
    int32_t doc_offset = 0;
    // Start with first document and no offset.
    sample_idx[2 * sample_index] = doc_idx_index;
    sample_idx[2 * sample_index + 1] = doc_offset;
    ++sample_index;

    while (sample_index <= num_samples) {
        // Start with a fresh sequence.
      int32_t remaining_seq_length = seq_length + 1;
      while (remaining_seq_length != 0) {
            // Get the document length.
	auto doc_id = doc_idx[doc_idx_index];
	auto doc_length = sizes[doc_id] - doc_offset;
	// And add it to the current sequence.
	remaining_seq_length -= doc_length;
	// If we have more than a full sequence, adjust offset and set
	// remaining length to zero so we return from the while loop.
	// Note that -1 here is for the same reason we have -1 in
	// `_num_epochs` calculations.
	if (remaining_seq_length <= 0) {
	  doc_offset += (remaining_seq_length + doc_length - 1);
	  remaining_seq_length = 0;
	} else {
	  // Otherwise, start from the begining of the next document.
	  ++doc_idx_index;
	  doc_offset = 0;
	}
      }
      // Record the sequence.
      sample_idx[2 * sample_index] = doc_idx_index;
      sample_idx[2 * sample_index + 1] = doc_offset;
      ++sample_index;
    }

    // Method to deallocate memory.
    py::capsule free_when_done(sample_idx, [](void *mem_) {
	int32_t *mem = reinterpret_cast<int32_t*>(mem_);
	delete[] mem;
      });

    // Return the numpy array.
    const auto byte_size = sizeof(int32_t);
    return py::array(std::vector<int64_t>{num_samples+1, 2}, // shape
                     {2*byte_size, byte_size}, // C-style contiguous strides
                     sample_idx, // the data pointer
                     free_when_done); // numpy array references
    
}


188
189
190
inline int32_t get_target_sample_len(const int32_t short_seq_ratio,
				     const int32_t max_length,
				     std::mt19937& rand32_gen) {
191
    /* Training sample length. */
192
193
194
    if (short_seq_ratio == 0) {
      return max_length;
    }
195
    const auto random_number = rand32_gen();
196
    if ((random_number % short_seq_ratio) == 0) {
197
      return 2 + random_number % (max_length - 1);
198
199
    }
    return max_length;
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
200
201
}

202

203
template<typename DocIdx>
204
205
206
py::array build_mapping_impl(const py::array_t<int64_t>& docs_,
                             const py::array_t<int32_t>& sizes_,
                             const int32_t num_epochs,
207
                             const uint64_t max_num_samples,
208
                             const int32_t max_seq_length,
209
                             const double short_seq_prob,
210
                             const int32_t seed,
211
212
			     const bool verbose,
			     const int32_t min_num_sent) {
213
214
215
216
217
218
219
220
    /* Build a mapping of (start-index, end-index, sequence-length) where
       start and end index are the indices of the sentences in the sample
       and sequence-length is the target sequence length.
    */

    // Consistency checks.
    assert(num_epochs > 0);
    assert(max_seq_length > 1);
221
    assert(short_seq_prob >= 0.0);
222
223
    assert(short_seq_prob <= 1.0);
    assert(seed > 0);
224
225
226
227

    // Remove bound checks.
    auto docs = docs_.unchecked<1>();
    auto sizes = sizes_.unchecked<1>();
228
229

    // For efficiency, convert probability to ratio. Note: rand() generates int.
230
231
232
233
    int32_t short_seq_ratio = 0;
    if (short_seq_prob > 0) {
      short_seq_ratio = static_cast<int32_t>(round(1.0 / short_seq_prob));
    }
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257

    if (verbose) {
        const auto sent_start_index = docs[0];
	const auto sent_end_index = docs[docs_.shape(0) - 1];
	const auto num_sentences = sent_end_index - sent_start_index;
	cout << "    using:" << endl << std::flush;
	cout << "     number of documents:            " << docs_.shape(0) - 1 <<
	  endl << std::flush;
	cout << "     sentences range:                [" << sent_start_index <<
	", " << sent_end_index << ")" << endl << std::flush;
	cout << "     total number of sentences:      " << num_sentences <<
	  endl << std::flush;
	cout << "     number of epochs:               " << num_epochs <<
	  endl << std::flush;
	cout << "     maximum number of samples:      " << max_num_samples <<
	  endl << std::flush;
	cout << "     maximum sequence length:        " << max_seq_length <<
	  endl << std::flush;
	cout << "     short sequence probability:     " << short_seq_prob <<
	endl << std::flush;
	cout << "     short sequence ration (1/prob): " << short_seq_ratio <<
	  endl << std::flush;
	cout << "     seed:                           " << seed << endl <<
	  std::flush;
258
    }
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
259

260
261
262
263
264
265
266
    // Mapping and it's length (1D).
    int64_t num_samples = -1;
    DocIdx* maps = NULL;

    // Perform two iterations, in the first iteration get the size
    // and allocate memory and in the second iteration populate the map.
    bool second = false;
267
    for (int32_t iteration=0; iteration<2; ++iteration) {
268
269

        // Set the seed so both iterations produce the same results.
270
        std::mt19937 rand32_gen(seed);
271
272

        // Set the flag on second iteration.
273
        second = (iteration == 1);
274
275

        // Counters:
276
277
        uint64_t empty_docs = 0;
        uint64_t one_sent_docs = 0;
278
	uint64_t long_sent_docs = 0;
279
280
281
282
283

        // Current map index.
        uint64_t map_index = 0;

        // For each epoch:
284
285
286
        for (int32_t epoch=0; epoch<num_epochs; ++epoch) {
            if (map_index >= max_num_samples) {
	        if (verbose && (!second)) {
287
		  cout << "    reached " << max_num_samples << " samples after "
288
289
		       << epoch << " epochs ..." << endl << std::flush;
		}
290
291
292
                break;
            }
            // For each document:
293
            for (int32_t doc=0; doc<(docs.shape(0) - 1); ++doc) {
294

295
                // Document sentences are in [sent_index_first, sent_index_last)
296
297
298
                const auto sent_index_first = docs[doc];
                const auto sent_index_last = docs[doc + 1];

299
300
                // At the begining of the document previous index is the
		// start index.
301
302
303
304
305
306
307
308
                auto prev_start_index = sent_index_first;

                // Remaining documents.
                auto num_remain_sent = sent_index_last - sent_index_first;

                // Some bookkeeping
                if ((epoch == 0) && (!second)) {
                    if (num_remain_sent == 0) {
309
		        ++empty_docs;
310
311
                    }
                    if (num_remain_sent == 1) {
312
		        ++one_sent_docs;
313
314
315
                    }
                }

316
		// Detect documents with long sentences.
317
318
319
320
321
322
323
324
325
326
327
328
329
330
		bool contains_long_sentence = false;
		if (num_remain_sent > 1) {
		    for (auto sent_index=sent_index_first;
			 sent_index < sent_index_last; ++sent_index) {
		        if (sizes[sent_index] > LONG_SENTENCE_LEN){
			    if ((epoch == 0) && (!second)) {
			        ++long_sent_docs;
			    }
			    contains_long_sentence = true;
			    break;
			}
		    }
		}

331
                // If we have more than two sentences.
332
                if ((num_remain_sent >= min_num_sent) && (!contains_long_sentence)) {
333
334

                    // Set values.
335
336
337
338
339
                    auto seq_len = int32_t{0};
                    auto num_sent = int32_t{0};
                    auto target_seq_len = get_target_sample_len(short_seq_ratio,
								max_seq_length,
								rand32_gen);
340
341
342
343
344

                    // Loop through sentences.
                    for (auto sent_index=sent_index_first;
                         sent_index < sent_index_last; ++sent_index) {

345
346
347
348
349
350
351
352
353
354
355
		        // Add the size and number of sentences.
		        seq_len += sizes[sent_index];
		        ++num_sent;
			--num_remain_sent;

			// If we have reached the target length.
			// and if not only one sentence is left in the document.
			// and if we have at least two sentneces.
			// and if we have reached end of the document.
			if (((seq_len >= target_seq_len) &&
			     (num_remain_sent > 1) &&
356
			     (num_sent >= min_num_sent) ) || (num_remain_sent == 0)) {
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386

			    // Check for overflow.
			    if ((3 * map_index + 2) >
				std::numeric_limits<int64_t>::max()) {
			        cout << "number of samples exceeded maximum "
				     << "allowed by type int64: "
				     << std::numeric_limits<int64_t>::max()
				     << endl;
				throw std::overflow_error("Number of samples");
			    }

			    // Populate the map.
			    if (second) {
			        const auto map_index_0 = 3 * map_index;
				maps[map_index_0] = static_cast<DocIdx>(prev_start_index);
				maps[map_index_0 + 1] = static_cast<DocIdx>(sent_index + 1);
				maps[map_index_0 + 2] = static_cast<DocIdx>(target_seq_len);
			    }

			    // Update indices / counters.
			    ++map_index;
			    prev_start_index = sent_index + 1;
			    target_seq_len = get_target_sample_len(short_seq_ratio,
								   max_seq_length,
								   rand32_gen);
			    seq_len = 0;
			    num_sent = 0;
			}

                    } // for (auto sent_index=sent_index_first; ...
387
388
389
390
391
                } // if (num_remain_sent > 1) {
            } // for (int doc=0; doc < num_docs; ++doc) {
        } // for (int epoch=0; epoch < num_epochs; ++epoch) {

        if (!second) {
392
	    if (verbose) {
393
	        cout << "   number of empty documents: " << empty_docs <<
394
		  endl << std::flush;
395
		cout << "   number of documents with one sentence: " <<
396
		  one_sent_docs << endl << std::flush;
397
398
		cout << "   number of documents with long sentences: " <<
		  long_sent_docs << endl << std::flush;
399
		cout << "   will create mapping for " << map_index <<
400
401
402
403
		  " samples" << endl << std::flush;
	    }
	    assert(maps == NULL);
	    assert(num_samples < 0);
404
            maps = new DocIdx[3*map_index];
405
            num_samples = static_cast<int64_t>(map_index);
406
407
408
409
410
        }

    } // for (int iteration=0; iteration < 2; ++iteration) {

    // Shuffle.
411
412
413
    // We need a 64 bit random number generator as we might have more
    // than 2 billion samples.
    std::mt19937_64 rand64_gen(seed + 1);
414
    for (auto i=(num_samples - 1); i > 0; --i) {
415
416
417
418
419
420
421
      const auto j = static_cast<int64_t>(rand64_gen() % (i + 1));
      const auto i0 = 3 * i;
      const auto j0 = 3 * j;
      // Swap values.
      swap(maps[i0], maps[j0]);
      swap(maps[i0 + 1], maps[j0 + 1]);
      swap(maps[i0 + 2], maps[j0 + 2]);
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
422
423
    }

424
425
426
    // Method to deallocate memory.
    py::capsule free_when_done(maps, [](void *mem_) {
            DocIdx *mem = reinterpret_cast<DocIdx*>(mem_);
427
	    delete[] mem;
428
        });
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
429

430
    // Return the numpy array.
431
    const auto byte_size = sizeof(DocIdx);
432
    return py::array(std::vector<int64_t>{num_samples, 3}, // shape
433
                     {3*byte_size, byte_size}, // C-style contiguous strides
434
435
                     maps, // the data pointer
                     free_when_done); // numpy array references
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
436

437
}
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
438

439
440
441

py::array build_mapping(const py::array_t<int64_t>& docs_,
                        const py::array_t<int>& sizes_,
442
443
444
445
                        const int num_epochs,
                        const uint64_t max_num_samples,
                        const int max_seq_length,
                        const double short_seq_prob,
446
                        const int seed,
447
448
			const bool verbose,
			const int32_t min_num_sent) {
449

450
    if (sizes_.size() > std::numeric_limits<uint32_t>::max()) {
451
        if (verbose) {
452
453
454
	   cout << "    using uint64 for data mapping..." << endl << std::flush;
	}
	return build_mapping_impl<uint64_t>(docs_, sizes_, num_epochs,
455
					    max_num_samples, max_seq_length,
456
457
					    short_seq_prob, seed, verbose,
					    min_num_sent);
458
    } else {
459
460
461
462
463
       if (verbose) {
	   cout << "    using uint32 for data mapping..." << endl << std::flush;
       }
       return build_mapping_impl<uint32_t>(docs_, sizes_, num_epochs,
					   max_num_samples, max_seq_length,
464
465
					   short_seq_prob, seed, verbose,
					   min_num_sent);
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
466
467
468
    }
}

469
470
471
472
473
474
475
476
template<typename DocIdx>
py::array build_blocks_mapping_impl(const py::array_t<int64_t>& docs_,
                                    const py::array_t<int32_t>& sizes_,
                                    const py::array_t<int32_t>& titles_sizes_,
                                    const int32_t num_epochs,
                                    const uint64_t max_num_samples,
                                    const int32_t max_seq_length,
                                    const int32_t seed,
477
478
                                    const bool verbose,
                                    const bool use_one_sent_blocks) {
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
    /* Build a mapping of (start-index, end-index, sequence-length) where
       start and end index are the indices of the sentences in the sample
       and sequence-length is the target sequence length.
    */

    // Consistency checks.
    assert(num_epochs > 0);
    assert(max_seq_length > 1);
    assert(seed > 0);

    // Remove bound checks.
    auto docs = docs_.unchecked<1>();
    auto sizes = sizes_.unchecked<1>();
    auto titles_sizes = titles_sizes_.unchecked<1>();

    if (verbose) {
        const auto sent_start_index = docs[0];
        const auto sent_end_index = docs[docs_.shape(0) - 1];
        const auto num_sentences = sent_end_index - sent_start_index;
        cout << "    using:" << endl << std::flush;
        cout << "     number of documents:            " << docs_.shape(0) - 1 <<
          endl << std::flush;
        cout << "     sentences range:                [" << sent_start_index <<
        ", " << sent_end_index << ")" << endl << std::flush;
        cout << "     total number of sentences:      " << num_sentences <<
          endl << std::flush;
        cout << "     number of epochs:               " << num_epochs <<
          endl << std::flush;
        cout << "     maximum number of samples:      " << max_num_samples <<
          endl << std::flush;
        cout << "     maximum sequence length:        " << max_seq_length <<
          endl << std::flush;
        cout << "     seed:                           " << seed << endl <<
          std::flush;
    }

    // Mapping and its length (1D).
    int64_t num_samples = -1;
    DocIdx* maps = NULL;

519
520
521
522
523
524
    // Acceptable number of sentences per block.
    int min_num_sent = 2;
    if (use_one_sent_blocks) {
        min_num_sent = 1;
    }

525
526
527
528
529
530
531
532
533
534
535
    // Perform two iterations, in the first iteration get the size
    // and allocate memory and in the second iteration populate the map.
    bool second = false;
    for (int32_t iteration=0; iteration<2; ++iteration) {

        // Set the flag on second iteration.
        second = (iteration == 1);

        // Current map index.
        uint64_t map_index = 0;

536
537
538
        uint64_t empty_docs = 0;
        uint64_t one_sent_docs = 0;
        uint64_t long_sent_docs = 0;
539
540
        // For each epoch:
        for (int32_t epoch=0; epoch<num_epochs; ++epoch) {
541
542
543
            // assign every block a unique id
            int32_t block_id = 0;

544
545
546
547
548
549
550
551
552
553
554
555
556
            if (map_index >= max_num_samples) {
                if (verbose && (!second)) {
                cout << "    reached " << max_num_samples << " samples after "
                     << epoch << " epochs ..." << endl << std::flush;
                }
                break;
            }
            // For each document:
            for (int32_t doc=0; doc<(docs.shape(0) - 1); ++doc) {

                // Document sentences are in [sent_index_first, sent_index_last)
                const auto sent_index_first = docs[doc];
                const auto sent_index_last = docs[doc + 1];
Neel Kant's avatar
Neel Kant committed
557
                const auto target_seq_len = max_seq_length - titles_sizes[doc];
558
559
560
561
562
563
564
565

                // At the begining of the document previous index is the
                // start index.
                auto prev_start_index = sent_index_first;

                // Remaining documents.
                auto num_remain_sent = sent_index_last - sent_index_first;

566
567
568
569
570
571
572
573
574
                // Some bookkeeping
                if ((epoch == 0) && (!second)) {
                    if (num_remain_sent == 0) {
		                ++empty_docs;
                    }
                    if (num_remain_sent == 1) {
		                ++one_sent_docs;
                    }
                }
575
576
                // Detect documents with long sentences.
                bool contains_long_sentence = false;
577
                if (num_remain_sent >= min_num_sent) {
578
579
580
                    for (auto sent_index=sent_index_first;
                    sent_index < sent_index_last; ++sent_index) {
                        if (sizes[sent_index] > LONG_SENTENCE_LEN){
581
582
583
                            if ((epoch == 0) && (!second)) {
                                ++long_sent_docs;
                            }
584
585
586
587
588
                            contains_long_sentence = true;
                            break;
                        }
                    }
                }
589
590
                // If we have enough sentences and no long sentences.
                if ((num_remain_sent >= min_num_sent) && (!contains_long_sentence)) {
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605

                    // Set values.
                    auto seq_len = int32_t{0};
                    auto num_sent = int32_t{0};

                    // Loop through sentences.
                    for (auto sent_index=sent_index_first;
                         sent_index < sent_index_last; ++sent_index) {

                            // Add the size and number of sentences.
                            seq_len += sizes[sent_index];
                            ++num_sent;
                            --num_remain_sent;

                        // If we have reached the target length.
606
607
                        // and there are an acceptable number of sentences left
                        // and if we have at least the minimum number of sentences.
608
609
                        // or if we have reached end of the document.
                        if (((seq_len >= target_seq_len) &&
610
611
                             (num_remain_sent >= min_num_sent) &&
                             (num_sent >= min_num_sent) ) || (num_remain_sent == 0)) {
612
613
614

                            // Populate the map.
                            if (second) {
Neel Kant's avatar
Neel Kant committed
615
                                const auto map_index_0 = 4 * map_index;
616
617
618
619
                                // Each sample has 4 items: the starting sentence index, ending sentence index,
                                // the index of the document from which the block comes (used for fetching titles)
                                // and the unique id of the block (used for creating block indexes)

620
621
                                maps[map_index_0] = static_cast<DocIdx>(prev_start_index);
                                maps[map_index_0 + 1] = static_cast<DocIdx>(sent_index + 1);
622
                                maps[map_index_0 + 2] = static_cast<DocIdx>(doc);
Neel Kant's avatar
Neel Kant committed
623
                                maps[map_index_0 + 3] = static_cast<DocIdx>(block_id);
624
625
626
627
                            }

                            // Update indices / counters.
                            ++map_index;
Neel Kant's avatar
Neel Kant committed
628
                            ++block_id;
629
630
631
632
633
634
635
636
637
638
639
                            prev_start_index = sent_index + 1;
                            seq_len = 0;
                            num_sent = 0;
                        }
                    } // for (auto sent_index=sent_index_first; ...
                } // if (num_remain_sent > 1) {
            } // for (int doc=0; doc < num_docs; ++doc) {
        } // for (int epoch=0; epoch < num_epochs; ++epoch) {

        if (!second) {
            if (verbose) {
640
641
642
643
644
645
	        cout << "   number of empty documents: " << empty_docs <<
              endl << std::flush;
            cout << "   number of documents with one sentence: " <<
              one_sent_docs << endl << std::flush;
            cout << "   number of documents with long sentences: " <<
              long_sent_docs << endl << std::flush;
646
647
648
649
650
            cout << "   will create mapping for " << map_index <<
              " samples" << endl << std::flush;
            }
            assert(maps == NULL);
            assert(num_samples < 0);
Neel Kant's avatar
Neel Kant committed
651
            maps = new DocIdx[4*map_index];
652
653
654
655
656
            num_samples = static_cast<int64_t>(map_index);
        }

    } // for (int iteration=0; iteration < 2; ++iteration) {

Neel Kant's avatar
Neel Kant committed
657
658
659
    // Shuffle.
    // We need a 64 bit random number generator as we might have more
    // than 2 billion samples.
660
661
662
    std::mt19937_64 rand64_gen(seed + 1);
    for (auto i=(num_samples - 1); i > 0; --i) {
        const auto j = static_cast<int64_t>(rand64_gen() % (i + 1));
Neel Kant's avatar
Neel Kant committed
663
664
        const auto i0 = 4 * i;
        const auto j0 = 4 * j;
665
666
667
668
        // Swap values.
        swap(maps[i0], maps[j0]);
        swap(maps[i0 + 1], maps[j0 + 1]);
        swap(maps[i0 + 2], maps[j0 + 2]);
Neel Kant's avatar
Neel Kant committed
669
        swap(maps[i0 + 3], maps[j0 + 3]);
670
671
672
673
674
675
676
677
678
679
    }

    // Method to deallocate memory.
    py::capsule free_when_done(maps, [](void *mem_) {
            DocIdx *mem = reinterpret_cast<DocIdx*>(mem_);
	    delete[] mem;
        });

    // Return the numpy array.
    const auto byte_size = sizeof(DocIdx);
Neel Kant's avatar
Neel Kant committed
680
681
    return py::array(std::vector<int64_t>{num_samples, 4}, // shape
                     {4*byte_size, byte_size}, // C-style contiguous strides
682
683
684
685
686
687
688
689
690
691
692
693
                     maps, // the data pointer
                     free_when_done); // numpy array references

}

py::array build_blocks_mapping(const py::array_t<int64_t>& docs_,
                               const py::array_t<int>& sizes_,
                               const py::array_t<int>& titles_sizes_,
                               const int num_epochs,
                               const uint64_t max_num_samples,
                               const int max_seq_length,
                               const int seed,
694
695
                    const bool verbose,
                    const bool use_one_sent_blocks) {
696
697
698
699
700
701

    if (sizes_.size() > std::numeric_limits<uint32_t>::max()) {
        if (verbose) {
	   cout << "    using uint64 for data mapping..." << endl << std::flush;
	}
	return build_blocks_mapping_impl<uint64_t>(docs_, sizes_, titles_sizes_,
702
	                    num_epochs, max_num_samples, max_seq_length, seed, verbose, use_one_sent_blocks);
703
704
705
706
707
    } else {
       if (verbose) {
	   cout << "    using uint32 for data mapping..." << endl << std::flush;
       }
       return build_blocks_mapping_impl<uint32_t>(docs_, sizes_, titles_sizes_,
708
                        num_epochs, max_num_samples, max_seq_length, seed, verbose, use_one_sent_blocks);
709
710
    }
}
711

Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
712
PYBIND11_MODULE(helpers, m) {
713
    m.def("build_mapping", &build_mapping);
Neel Kant's avatar
Neel Kant committed
714
    m.def("build_blocks_mapping", &build_blocks_mapping);
Mohammad's avatar
Mohammad committed
715
    m.def("build_sample_idx", &build_sample_idx);
mohammad's avatar
mohammad committed
716
    m.def("build_blending_indices", &build_blending_indices);
Mohammad Shoeybi's avatar
Mohammad Shoeybi committed
717
}