"examples/nlp/vscode:/vscode.git/clone" did not exist on "b43562a74d60bf5c674895ce83c30d5130ed1308"
instance.py 1.17 KB
Newer Older
1
from dataclasses import dataclass, field
2
3
from typing import Literal, Optional, Tuple

Baber's avatar
nit  
Baber committed
4
5
from lm_eval.api.types import GenerateUntilRequest, LogLiklehoodRequest

6
7
8
9

OutputType = Literal[
    "loglikelihood", "loglikelihood_rolling", "generate_until", "multiple_choice"
]
10

lintangsutawika's avatar
lintangsutawika committed
11

12
13
@dataclass
class Instance:
14
    request_type: OutputType
haileyschoelkopf's avatar
haileyschoelkopf committed
15
16
17
    doc: dict
    arguments: tuple
    idx: int
18
    metadata: Tuple[Optional[str], Optional[int], Optional[int]] = field(
haileyschoelkopf's avatar
haileyschoelkopf committed
19
        default_factory=lambda: (None, None, None)
20
    )
21
22
23
    resps: list = field(default_factory=list)
    filtered_resps: dict = field(default_factory=dict)

24
    # initialized after init
25
26
27
    task_name: Optional[str] = None
    doc_id: Optional[int] = None
    repeats: Optional[int] = None
28

Ethan Smith's avatar
Ethan Smith committed
29
    def __post_init__(self) -> None:
haileyschoelkopf's avatar
haileyschoelkopf committed
30
        # unpack metadata field
31
        self.task_name, self.doc_id, self.repeats = self.metadata
lintangsutawika's avatar
lintangsutawika committed
32

33
34
35
36
37
    @property
    def args(self):
        """
        Returns (string,) where `string` is the string to calculate loglikelihood over
        """
lintangsutawika's avatar
lintangsutawika committed
38
        return (
Baber's avatar
nit  
Baber committed
39
40
41
            GenerateUntilRequest(*self.arguments)
            if self.request_type != "generate_until"
            else LogLiklehoodRequest(*self.arguments)
lintangsutawika's avatar
lintangsutawika committed
42
        )