"router/client/src/v3/client.rs" did not exist on "e520d5b34917f02504a280f87420fd2a5ce0eeb2"
evaluate_orqa.py 946 Bytes
Newer Older
Jared Casper's avatar
Jared Casper committed
1
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
Mostofa Patwary's avatar
Mostofa Patwary committed
2
3
4

"""Main tasks functionality."""

xingjinliang's avatar
xingjinliang committed
5
6
from megatron.training import get_args, print_rank_0
from megatron.legacy.indexer import IndexBuilder
Mostofa Patwary's avatar
Mostofa Patwary committed
7
8
9
10
11
12
from tasks.orqa.evaluate_utils import ORQAEvaluator

def main():
    """
    Main program
    """
13

Mostofa Patwary's avatar
Mostofa Patwary committed
14
15
    args = get_args()

Mostofa Patwary's avatar
Mostofa Patwary committed
16
17
18
    """
    Create a BlockData data structure by running an IndexBuilder over an
    ICT Dataset and then evaluate on NQ task
Mostofa Patwary's avatar
Mostofa Patwary committed
19
20
    """

Mostofa Patwary's avatar
Mostofa Patwary committed
21
    print_rank_0("Starting index builder!")
22

Mostofa Patwary's avatar
Mostofa Patwary committed
23
24
25
26
    index_builder = IndexBuilder()
    index_builder.build_and_save_index()
    print_rank_0("Build and save indices: done!")

Mostofa Patwary's avatar
Mostofa Patwary committed
27
28
29

    print_rank_0("Starting evaluations!")

Mostofa Patwary's avatar
Mostofa Patwary committed
30
31
32
33
34
35
36
37
38
    # Set up the model and evaluator
    evaluator = ORQAEvaluator()

    # Run evaluation
    if args.qa_data_dev is not None:
        evaluator.evaluate(args.qa_data_dev, "DEV")

    if args.qa_data_test is not None:
        evaluator.evaluate(args.qa_data_test, "TEST")
Mostofa Patwary's avatar
Mostofa Patwary committed
39