Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
template.py 2.53 KB
Newer Older
Baber's avatar
refactor configs to files  
Baber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81 
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Callable, Optional, Union


if TYPE_CHECKING:
    from lm_eval.config.metric import MetricConfig


@dataclass
class TemplateConfig:
    """Encapsulates information about a template."""

    template: str
    doc_to_text: Union[str, Callable[[dict], str]]
    doc_to_choice: Union[str, list, Callable[[dict], list]]
    doc_to_target: Union[int, Callable[[dict], int]]
    description: str
    context_prefix: str
    prefix_delimiter: str
    context_delimiter: str
    answer_suffix: str
    target_delimiter: str
    choice_format: Optional[str]
    choice_delimiter: Optional[str]
    fewshot_delimiter: str
    metric_list: Optional[Union[list[str], list["MetricConfig"]]] = field(
        default_factory=lambda: ["acc", "acc_norm"]
    )


@dataclass
class MCQTemplateConfig:
    """Encapsulates information about a template.
    Would return a sample with the following format:
    Question: <doc_to_text(doc)>
    A. <doc_to_choice(doc)[0]>
    B. <doc_to_choice(doc)[1]>
    C. <doc_to_choice(doc)[2]>
    D. <doc_to_choice(doc)[3]>
    Answer:` doc_to_choice(doc)` for each choice.
    """

    doc_to_text: Union[str, Callable[[dict], str]]
    doc_to_choice: Union[str, list, Callable[[dict], list]]
    doc_to_target: Union[int, Callable[[dict], int]]
    template = "mcq"
    context_prefix: str = "Question:"
    prefix_delimiter: str = " "
    context_delimiter: str = "\n"
    answer_suffix: str = "Answer:"
    target_delimiter: str = "\n"
    choice_format: Optional[str] = "letters"
    choice_delimiter: Optional[str] = "\n"
    fewshot_delimiter: str = "\n\n"
    metric_list: Optional[list["MetricConfig"]] = field(default_factory=lambda: ["acc"])


@dataclass
class ClozeTemplateConfig:
    """Encapsulates information about a template.
    Would return a sample with the following format:
    Question:  <doc_to_text(doc)>
    Answer:` <doc_to_target(doc)>`
    """

    doc_to_text: Union[str, Callable[[dict], str]]
    doc_to_choice: Union[str, list, Callable[[dict], list]]
    doc_to_target: Union[int, Callable[[dict], int]]
    template: str = "cloze"
    description: str = ""
    context_prefix: str = "Question:"
    prefix_delimiter: str = " "
    context_delimiter: str = "\n"
    answer_suffix: str = "Answer:"
    target_delimiter: str = " "
    choice_format: Optional[str] = None
    choice_delimiter: Optional[str] = None
    fewshot_delimiter: str = "\n\n"
    metric_list: Optional[list["MetricConfig"]] = field(
        default_factory=lambda: ["acc", "acc_norm"]
    )