Merge branch 'main_retriver_merge' into main_retriver_merge_ict_eval

4bea4e76 · Mostofa Patwary · 613d0fe8 · 22a3d81a · 4bea4e76
Commit 4bea4e76 authored Feb 11, 2021 by Mostofa Patwary
Hide whitespace changes
Inline Side-by-side

Showing with 0 additions and 4 deletions

megatron/data/biencoder_dataset_utils.py megatron/data/biencoder_dataset_utils.py +0 -4

No files found.
--- a/megatron/data/biencoder_dataset_utils.py
+++ b/megatron/data/biencoder_dataset_utils.py
@@ -125,10 +125,6 @@ def get_block_samples_mapping(block_dataset, title_dataset, data_prefix, num_epo
        print_rank_0(' > building samples index mapping for {} ...'.format(
            name))

-        # compile/bind the C++ helper code
-        from megatron.data.dataset_utils import compile_helper
-        compile_helper()
-
        from megatron.data import helpers
        mapping_array = helpers.build_blocks_mapping(
            block_dataset.doc_idx,