"megatron/legacy/model/language_model.py" did not exist on "c2ea914f7df6e74262375bffb0e9b112ca612a00"
naturalqs.py 3.61 KB
Newer Older
1
from . common import HFTask
Leo Gao's avatar
Leo Gao committed
2
from itertools import islice
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

class NaturalQs(HFTask):
    DATASET_PATH = "natural_questions"
    DATASET_NAME = None

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return False

    def fewshot_description(self):
        # TODO: figure out description
        return ""

21
22
23
    def training_docs(self):
        # Cache training for faster few-shot.
        # Data is too large to fit in memory.
Charles Foster's avatar
Charles Foster committed
24
        return self.data["train"]
25

Leo Gao's avatar
Leo Gao committed
26
27
28
29
30
31
32
    def fewshot_examples(self, k):
        # Data is too large to fit in memory. We just sample from the first bit.
        if self._traindocs is None:
            self._traindocs = list(islice(self.training_docs(), 0, 100000))

        return random.sample(self._traindocs, k)

33
34
35
36
37
38
39
40
41
42
43
44
45
    def doc_to_text(self, doc):
        return 'Q: ' + doc['question']['text'] + '\n\n' + 'A: '

    def doc_to_target(self, doc):
        # There's a short answer and a long answer. Based on the paper, I'm using the long answer.
        short_answer = doc['annotations']['short_answers'][0]['text']
        long_answer_start = doc['annotations']['long_answer'][0]['start_token']
        long_answer_end = doc['annotations']['long_answer'][0]['end_token']
        long_answer_span = doc['document']['tokens']['token'][long_answer_start:long_answer_end]
        long_answer_is_html = doc['document']['tokens']['is_html'][long_answer_start:long_answer_end]
        long_answer_chars = [tok for (tok, is_html) in zip(long_answer_span, long_answer_is_html) if not is_html]
        long_answer = " ".join(long_answer_chars)
        return long_answer # Replace with short_answer[0] for short answer
46

Leo Gao's avatar
Leo Gao committed
47
48
49
    def construct_requests(self, doc, ctx):
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.
50

Leo Gao's avatar
Leo Gao committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')
    
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a 
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
Leo Gao's avatar
Leo Gao committed
90
        raise NotImplementedError('Evaluation not implemented')