task.py 52.9 KB
Newer Older
1
import abc
2
import ast
lintangsutawika's avatar
lintangsutawika committed
3
import logging
4
import random
5
6
7
import re
from collections.abc import Callable
from dataclasses import asdict, dataclass
8
from inspect import getsource
9
from typing import Any, List, Literal, Tuple, Union
10
11
12
13
14

import datasets
import numpy as np

from lm_eval import utils
15
from lm_eval.api import samplers
haileyschoelkopf's avatar
haileyschoelkopf committed
16
from lm_eval.api.instance import Instance
lintangsutawika's avatar
lintangsutawika committed
17
from lm_eval.api.metrics import (
18
    bits_per_byte,
lintangsutawika's avatar
lintangsutawika committed
19
20
21
22
    mean,
    weighted_perplexity,
)
from lm_eval.api.registry import (
23
24
    AGGREGATION_REGISTRY,
    DEFAULT_METRIC_REGISTRY,
haileyschoelkopf's avatar
haileyschoelkopf committed
25
    get_aggregation,
26
    get_metric,
27
    get_metric_aggregation,
haileyschoelkopf's avatar
haileyschoelkopf committed
28
    is_higher_better,
lintangsutawika's avatar
lintangsutawika committed
29
)
30
31
32
from lm_eval.filters import build_filter_ensemble
from lm_eval.prompts import get_prompt

33

34
35
36
37
ALL_OUTPUT_TYPES = [
    "loglikelihood",
    "multiple_choice",
    "loglikelihood_rolling",
38
    "generate_until",
39
40
]

41
eval_logger = logging.getLogger("lm-eval")
42

lintangsutawika's avatar
lintangsutawika committed
43

lintangsutawika's avatar
lintangsutawika committed
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
@dataclass
class GroupConfig(dict):
    group: str = None
    task: Union[str, list] = None
    weight_by_size: bool = False

    def __getitem__(self, item):
        return getattr(self, item)

    def __setitem__(self, item, value):
        return setattr(self, item, value)

    def to_dict(self):
        return asdict(self)

lintangsutawika's avatar
lintangsutawika committed
59

60
61
@dataclass
class TaskConfig(dict):
62
    # task naming/registry
63
    task: str = None
lintangsutawika's avatar
lintangsutawika committed
64
    task_alias: str = None
65
    group: Union[str, list] = None
lintangsutawika's avatar
lintangsutawika committed
66
    group_alias: Union[str, list] = None
67
68
69
    # HF dataset options.
    # which dataset to use,
    # and what splits for what purpose
70
71
    dataset_path: str = None
    dataset_name: str = None
72
    dataset_kwargs: dict = None
73
74
75
    training_split: str = None
    validation_split: str = None
    test_split: str = None
lintangsutawika's avatar
lintangsutawika committed
76
    fewshot_split: str = None  # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaling (?)
77
78
    # formatting / prompting options.
    # see docs/advanced_task_guide.md for more info
79
    process_docs: Callable = None
80
81
    doc_to_text: Union[Callable, str] = None
    doc_to_target: Union[Callable, str] = None
lintangsutawika's avatar
lintangsutawika committed
82
    doc_to_choice: Union[Callable, str, dict, list] = None
lintangsutawika's avatar
lintangsutawika committed
83
    process_results: Union[Callable, str] = None
84
    use_prompt: str = None
85
    description: str = ""
86
87
    target_delimiter: str = " "
    fewshot_delimiter: str = "\n\n"
haileyschoelkopf's avatar
haileyschoelkopf committed
88
    fewshot_config: dict = None
89
    # runtime configuration options
90
    num_fewshot: int = None
91
    # scoring options
92
    metric_list: list = None
93
94
95
96
97
98
    output_type: Literal[
        "loglikelihood",
        "loglikelihood_rolling",
        "generate_until",
        "multiple_choice",
    ] = "generate_until"
99
    generation_kwargs: dict = None
100
    repeats: int = 1
lintangsutawika's avatar
lintangsutawika committed
101
    filter_list: Union[str, list] = None
102
103
    should_decontaminate: bool = False
    doc_to_decontamination_query: str = None
104
    metadata: dict = None  # by default, not used in the code. allows for users to pass arbitrary info to tasks
105

Ethan Smith's avatar
Ethan Smith committed
106
    def __post_init__(self) -> None:
Lintang Sutawika's avatar
Lintang Sutawika committed
107
        if self.generation_kwargs is not None:
108
            if self.output_type != "generate_until":
Lintang Sutawika's avatar
Lintang Sutawika committed
109
                eval_logger.warning(
110
                    f"[{self.task}] passed `generation_kwargs`, but not using `output_type: generate_until`!"
Lintang Sutawika's avatar
Lintang Sutawika committed
111
                )
112
                assert self.output_type != "generate_until"
Lintang Sutawika's avatar
Lintang Sutawika committed
113
114
115
116
117
118
119

            if "temperature" in self.generation_kwargs:
                self.generation_kwargs["temperature"] = float(
                    self.generation_kwargs["temperature"]
                )

            if "until" not in self.generation_kwargs:
120
                self.generation_kwargs["until"] = [self.fewshot_delimiter]
Lintang Sutawika's avatar
Lintang Sutawika committed
121
        else:
122
            if self.output_type == "generate_until":
Lintang Sutawika's avatar
Lintang Sutawika committed
123
124
                # ensure that we greedily generate in absence of explicit arguments otherwise
                self.generation_kwargs = {
Lintang Sutawika's avatar
Lintang Sutawika committed
125
                    "until": None
126
127
                    if self.fewshot_delimiter is None
                    else [self.fewshot_delimiter],
Lintang Sutawika's avatar
Lintang Sutawika committed
128
129
                    "do_sample": False,
                }
130

131
132
133
    def __getitem__(self, item):
        return getattr(self, item)

134
135
136
    def __setitem__(self, item, value):
        return setattr(self, item, value)

137
    def to_dict(self, keep_callable: bool = False) -> dict:
138
139
        """dumps the current config as a dictionary object, as a printable format.
        null fields will not be printed.
haileyschoelkopf's avatar
haileyschoelkopf committed
140
        Used for dumping results alongside full task configuration
141

haileyschoelkopf's avatar
haileyschoelkopf committed
142
143
144
145
146
147
148
149
150
151
        :return: dict
            A printable dictionary version of the TaskConfig object.

        # TODO: should any default value in the TaskConfig not be printed?
        """
        cfg_dict = asdict(self)
        # remove values that are `None`
        for k, v in list(cfg_dict.items()):
            if v is None:
                cfg_dict.pop(k)
152
153
154
155
156
157
158
159
160
161
            elif k == "metric_list":
                for metric_dict in v:
                    for metric_key, metric_value in metric_dict.items():
                        if callable(metric_value):
                            metric_dict[metric_key] = self.serialize_function(
                                metric_value, keep_callable=keep_callable
                            )
                cfg_dict[k] = v
            elif callable(v):
                cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable)
haileyschoelkopf's avatar
haileyschoelkopf committed
162
        return cfg_dict
163

164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
    def serialize_function(
        self, value: Union[Callable, str], keep_callable=False
    ) -> Union[Callable, str]:
        """Serializes a given function or string.

        If 'keep_callable' is True, the original callable is returned.
        Otherwise, attempts to return the source code of the callable using 'getsource'.
        """
        if keep_callable:
            return value
        else:
            try:
                return getsource(value)
            except (TypeError, OSError):
                return str(value)

180
181
182
183
184
185
186
187
188
189
190
191

class Task(abc.ABC):
    """A task represents an entire benchmark including its dataset, problems,
    answers, and evaluation methods. See BoolQ for a simple example implementation

    A `doc` can be any python object which represents one instance of evaluation.
    This is usually a dictionary e.g.
        {"question": ..., "answer": ...} or
        {"question": ..., question, answer)
    """

    VERSION = None
192

193
194
195
196
197
198
199
200
    # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub
    # or a path to a custom `datasets` loading script.
    DATASET_PATH: str = None

    # The name of a subset within `DATASET_PATH`.
    DATASET_NAME: str = None

    OUTPUT_TYPE: str = None
lintangsutawika's avatar
lintangsutawika committed
201

202
203
204
205
206
207
    def __init__(
        self,
        data_dir=None,
        cache_dir=None,
        download_mode=None,
        config=None,
Ethan Smith's avatar
Ethan Smith committed
208
    ) -> None:
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
        """
        :param data_dir: str
            Stores the path to a local folder containing the `Task`'s data files.
            Use this to specify the path to manually downloaded data (usually when
            the dataset is not publicly accessible).
        :param cache_dir: str
            The directory to read/write the `Task` dataset. This follows the
            HuggingFace `datasets` API with the default cache directory located at:
                `~/.cache/huggingface/datasets`
            NOTE: You can change the cache location globally for a given process
            to another directory:
                `export HF_DATASETS_CACHE="/path/to/another/directory"`
        :param download_mode: datasets.DownloadMode
            How to treat pre-existing `Task` downloads and data.
            - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS`
                Reuse download and reuse dataset.
            - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS`
                Reuse download with fresh dataset.
            - `datasets.DownloadMode.FORCE_REDOWNLOAD`
                Fresh download and fresh dataset.
        """
        self.download(data_dir, cache_dir, download_mode)
        self._training_docs = None
        self._fewshot_docs = None
        self._instances = None

lintangsutawika's avatar
lintangsutawika committed
235
        self._config = TaskConfig({**config}) if config else TaskConfig()
236

lintangsutawika's avatar
lintangsutawika committed
237
        self._filters = [build_filter_ensemble("none", [["take_first", None]])]
238

Ethan Smith's avatar
Ethan Smith committed
239
    def download(self, data_dir=None, cache_dir=None, download_mode=None) -> None:
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
        """Downloads and returns the task dataset.
        Override this method to download the dataset from a custom API.

        :param data_dir: str
            Stores the path to a local folder containing the `Task`'s data files.
            Use this to specify the path to manually downloaded data (usually when
            the dataset is not publicly accessible).
        :param cache_dir: str
            The directory to read/write the `Task` dataset. This follows the
            HuggingFace `datasets` API with the default cache directory located at:
                `~/.cache/huggingface/datasets`
            NOTE: You can change the cache location globally for a given process
            by setting the shell environment variable, `HF_DATASETS_CACHE`,
            to another directory:
                `export HF_DATASETS_CACHE="/path/to/another/directory"`
        :param download_mode: datasets.DownloadMode
            How to treat pre-existing `Task` downloads and data.
            - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS`
                Reuse download and reuse dataset.
            - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS`
                Reuse download with fresh dataset.
            - `datasets.DownloadMode.FORCE_REDOWNLOAD`
                Fresh download and fresh dataset.
        """
264
265
266
267
268
269
270
        self.dataset = datasets.load_dataset(
            path=self.DATASET_PATH,
            name=self.DATASET_NAME,
            data_dir=data_dir,
            cache_dir=cache_dir,
            download_mode=download_mode,
        )
271

272
273
274
275
276
    @property
    def config(self):
        """Returns the TaskConfig associated with this class."""
        return self._config

277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    @abc.abstractmethod
    def has_training_docs(self):
        """Whether the task has a training set"""
        pass

    @abc.abstractmethod
    def has_validation_docs(self):
        """Whether the task has a validation set"""
        pass

    @abc.abstractmethod
    def has_test_docs(self):
        """Whether the task has a test set"""
        pass

    def training_docs(self):
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
        return []

    def validation_docs(self):
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
        return []

    def test_docs(self):
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
        return []

313
314
315
316
317
318
319
320
321
322
    def fewshot_docs(self):
        """
        :return: Iterable[obj]
            A iterable of any object, that doc_to_text can handle
        """
        if self.has_training_docs():
            return self.training_docs()
        elif self.has_validation_docs():
            return self.validation_docs()
        else:
lintangsutawika's avatar
lintangsutawika committed
323
            eval_logger.warning(
324
                f"[Task: {self.config.task}] has_training_docs and has_validation_docs are False"
325
                ", using test_docs as fewshot_docs but this is not recommended."
lintangsutawika's avatar
lintangsutawika committed
326
            )
327
328
            return self.test_docs()

329
330
331
332
333
334
335
336
337
338
    def _process_doc(self, doc):
        """
        Override this to process (detokenize, strip, replace, etc.) individual
        documents. This can be used in a map over documents of a data split.
        E.g. `map(self._process_doc, self.dataset["validation"])`

        :return: dict
            The processed version of the specified `doc`.
        """
        return doc
lintangsutawika's avatar
lintangsutawika committed
339

340
341
342
343
344
345
346
347
348
349
350
351
352
    @property
    def instances(self):
        """After calling `task.build_all_requests()`, tasks
        maintain a list of the dataset instances which will be evaluated.
        """
        return self._instances

    def fewshot_examples(self, k, rnd):
        if self._training_docs is None:
            self._training_docs = list(self.training_docs())

        return rnd.sample(self._training_docs, k)

Ethan Smith's avatar
Ethan Smith committed
353
    def doc_to_decontamination_query(self, doc) -> None:
354
355
356
357
358
359
360
361
362
363
364
365
366
        print(
            "Override doc_to_decontamination_query with document specific decontamination query."
        )
        assert False

    @abc.abstractmethod
    def doc_to_text(self, doc):
        pass

    @abc.abstractmethod
    def doc_to_target(self, doc):
        pass

Ethan Smith's avatar
Ethan Smith committed
367
    def build_all_requests(self, limit=None, rank=None, world_size=None) -> None:
368
369
370
371
372
373
        """Build a set of Instances for a task, and store them in task.instances"""
        if self.has_test_docs():
            docs = self.test_docs()
        elif self.has_validation_docs():
            docs = self.validation_docs()
        else:
374
            assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!"
375

376
        eval_logger.info(f"Building contexts for task on rank {rank}...")
377

378
        instances = []
379
380
        for doc_id, doc in utils.create_iterator(
            enumerate(docs), rank, world_size, limit
lintangsutawika's avatar
lintangsutawika committed
381
        ):
382
            # sample fewshot context #TODO: need to offset doc_id by rank now!
383
            fewshot_ctx = self.fewshot_context(
384
                doc,
385
                0 if self.config.num_fewshot is None else self.config.num_fewshot,
386
            )
387

388
            # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute
lintangsutawika's avatar
lintangsutawika committed
389
390
391
            inst = self.construct_requests(
                doc=doc,
                ctx=fewshot_ctx,
392
                metadata=(self.config["task"], doc_id, self.config.repeats),
lintangsutawika's avatar
lintangsutawika committed
393
            )
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418

            if not isinstance(inst, list):
                inst = [inst]

            instances.extend(inst)

        self._instances = instances
        assert len(self._instances) != 0, "task.build_requests() did not find any docs!"

    @abc.abstractmethod
    def construct_requests(self, doc, ctx, **kwargs):
        """Uses RequestFactory to construct Requests and returns an iterable of
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural
            language description, as well as the few shot examples, and the question
            part of the document for `doc`.
        :param doc_idx: int
            The index of a document within `self.test_docs()` or `self.validation_docs()`,
            whichever is the main split used.
        :param repeats: int
        TODO: update this docstring
lintangsutawika's avatar
lintangsutawika committed
419
            The number of times each instance in a dataset is inferred on. Defaults to 1,
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
            can be increased for techniques like majority voting.
        """
        pass

    @abc.abstractmethod
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a
        dict where keys are the names of submetrics and values are the values of
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        pass

    @abc.abstractmethod
    def aggregation(self):
        """
        :returns: {str: [metric_score] -> float}
            A dictionary where keys are the names of submetrics and values are
            functions that aggregate a list of metric scores
        """
        pass

    @abc.abstractmethod
    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are
            whether a higher value of the submetric is better
        """
        pass

455
456
457
    def get_config(self, key: str) -> Any:
        return getattr(self._config, key, None)

haileyschoelkopf's avatar
haileyschoelkopf committed
458
459
460
461
462
463
464
465
466
467
    @classmethod
    def count_bytes(cls, doc):
        """Used for byte-level perplexity metrics in rolling loglikelihood"""
        return len(doc.encode("utf-8"))

    @classmethod
    def count_words(cls, doc):
        """Downstream loglikelihood_rolling perplexity tasks with custom word boundaries should override this!"""
        return len(re.split(r"\s+", doc))

468
    @utils.positional_deprecated
lintangsutawika's avatar
lintangsutawika committed
469
    def fewshot_context(
470
471
472
473
474
        self,
        doc,
        num_fewshot,
        rnd=random.Random(1234),
        description=None,
lintangsutawika's avatar
lintangsutawika committed
475
    ):
476
477
478
479
480
481
482
        """Returns a fewshot context string that is made up of a prepended description
        (if provided), the `num_fewshot` number of examples, and an appended prompt example.

        :param doc: str
            The document as returned from training_docs, validation_docs, or test_docs.
        :param num_fewshot: int
            The number of fewshot examples to provide in the returned context string.
lintangsutawika's avatar
lintangsutawika committed
483
484
485
486
487
        :param rnd: random.Random
            The pseudo-random number generator used to randomly sample examples.
            WARNING: This is currently a required arg although it's optionalized with a default `None`.
        :param description: str
            The task's description that will be prepended to the fewshot examples.
488
489
490
        :returns: str
            The fewshot context.
        """
lintangsutawika's avatar
lintangsutawika committed
491
492
493
494
        assert (
            rnd is not None
        ), "A `random.Random` generator argument must be provided to `rnd`"

495
        description = description if description else ""
496
497

        if num_fewshot == 0:
lintangsutawika's avatar
lintangsutawika committed
498
            labeled_examples = ""
499
        else:
lintangsutawika's avatar
lintangsutawika committed
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
            # for sets with no training docs, draw from other set *but ensure no overlap with current doc*
            if self.has_training_docs():
                fewshotex = self.fewshot_examples(k=num_fewshot, rnd=rnd)
            else:
                if self._fewshot_docs is None:
                    self._fewshot_docs = list(
                        self.validation_docs()
                        if self.has_validation_docs()
                        else self.test_docs()
                    )

                fewshotex = rnd.sample(self._fewshot_docs, num_fewshot + 1)

                # get rid of the doc that's the one we're evaluating, if it's in the fewshot
                fewshotex = [x for x in fewshotex if x != doc][:num_fewshot]

            labeled_examples = (
                "\n\n".join(
                    [
                        self.doc_to_text(doc) + self.doc_to_target(doc)
                        for doc in fewshotex
                    ]
                )
                + "\n\n"
lintangsutawika's avatar
lintangsutawika committed
524
            )
525
526

        example = self.doc_to_text(doc)
lintangsutawika's avatar
lintangsutawika committed
527
        return description + labeled_examples + example
528
529

    def apply_filters(self):
lintangsutawika's avatar
lintangsutawika committed
530
531
        if hasattr(self, "_filters"):
            for f in self._filters:
532
                f.apply(self._instances)
lintangsutawika's avatar
lintangsutawika committed
533
534
535
        else:
            eval_logger.warning("No filter defined, passing through instances")
            return self._instances
536

baberabb's avatar
baberabb committed
537
    def dump_config(self) -> dict:
538
        """Returns a dictionary representing the task's config.
539
540
541
542
543

        :returns: str
            The fewshot context.
        """
        # TODO: this should only return the overrides applied to a non-YAML task's configuration.
544
        # (num_fewshot)
545
        return self.config.to_dict()
546

547
548

class ConfigurableTask(Task):
549
    VERSION = "Yaml"
550
    OUTPUT_TYPE = None
551
    CONFIG = None
552
553
554

    def __init__(
        self, data_dir=None, cache_dir=None, download_mode=None, config: dict = None
Ethan Smith's avatar
Ethan Smith committed
555
    ) -> None:  # TODO no super() call here
556
        # Get pre-configured attributes
557
        self._config = self.CONFIG
558

559
        # Use new configurations if there was no preconfiguration
560
        if self.config is None:
561
            self._config = TaskConfig(**config)
562
563
        # Overwrite configs
        else:
lintangsutawika's avatar
lintangsutawika committed
564
            if config is not None:
565
                self._config.__dict__.update(config)
566

567
        if self.config is None:
lintangsutawika's avatar
lintangsutawika committed
568
569
570
            raise ValueError(
                "Must pass a config to ConfigurableTask, either in cls.CONFIG or `config` kwarg"
            )
571

572
573
574
575
        if isinstance(self.config.metadata, dict):
            if "version" in self.config.metadata:
                self.VERSION = self.config.metadata["version"]

576
577
578
        if self.config.output_type is not None:
            assert self.config.output_type in ALL_OUTPUT_TYPES
            self.OUTPUT_TYPE = self.config.output_type
579

580
581
        if self.config.dataset_path is not None:
            self.DATASET_PATH = self.config.dataset_path
582

583
584
        if self.config.dataset_name is not None:
            self.DATASET_NAME = self.config.dataset_name
585

586
587
588
589
        self._metric_fn_list = {}
        self._metric_fn_kwargs = {}
        self._aggregation_list = {}
        self._higher_is_better = {}
590

591
        if self.config.metric_list is None:
592
            # TODO: handle this in TaskConfig.__post_init__ ?
593
594
            _metric_list = DEFAULT_METRIC_REGISTRY[self.config.output_type]

595
            for metric_name in _metric_list:
haileyschoelkopf's avatar
haileyschoelkopf committed
596
                self._metric_fn_list[metric_name] = get_metric(metric_name)
lintangsutawika's avatar
lintangsutawika committed
597
                self._metric_fn_kwargs[metric_name] = {}
598
599
600
                self._aggregation_list[metric_name] = get_metric_aggregation(
                    metric_name
                )
haileyschoelkopf's avatar
haileyschoelkopf committed
601
                self._higher_is_better[metric_name] = is_higher_better(metric_name)
602
        else:
603
            for metric_config in self.config.metric_list:
604
605
606
607
608
                assert "metric" in metric_config
                metric_name = metric_config["metric"]
                kwargs = {
                    key: metric_config[key]
                    for key in metric_config
Chris's avatar
Chris committed
609
610
                    if key
                    not in ["metric", "aggregation", "higher_is_better", "hf_evaluate"]
611
                }
Chris's avatar
Chris committed
612
613
614
615
                hf_evaluate_metric = (
                    "hf_evaluate" in metric_config
                    and metric_config["hf_evaluate"] is True
                )
616

617
                if self.config.process_results is not None:
618
619
                    self._metric_fn_list[metric_name] = None
                    self._metric_fn_kwargs[metric_name] = {}
620
621
622
623
624
625
                elif callable(metric_name):
                    metric_fn = metric_name.__call__
                    metric_name = metric_name.__name__
                    self._metric_fn_list[metric_name] = metric_fn
                    self._metric_fn_kwargs[metric_name] = kwargs
                else:
Chris's avatar
Chris committed
626
627
628
                    self._metric_fn_list[metric_name] = get_metric(
                        metric_name, hf_evaluate_metric
                    )
629
                    self._metric_fn_kwargs[metric_name] = kwargs
lintangsutawika's avatar
lintangsutawika committed
630

631
                if "aggregation" in metric_config:
632
                    agg_name = metric_config["aggregation"]
633
                    if isinstance(agg_name, str):
haileyschoelkopf's avatar
haileyschoelkopf committed
634
                        self._aggregation_list[metric_name] = get_aggregation(agg_name)
635
                    elif callable(agg_name):  # noqa: E721
636
637
638
                        self._aggregation_list[metric_name] = metric_config[
                            "aggregation"
                        ]
639
                else:
640
                    INV_AGG_REGISTRY = {v: k for k, v in AGGREGATION_REGISTRY.items()}
lintangsutawika's avatar
lintangsutawika committed
641
                    metric_agg = get_metric_aggregation(metric_name)
642
                    eval_logger.warning(
643
                        f"[Task: {self.config.task}] metric {metric_name} is defined, but aggregation is not. "
644
645
                        f"using default "
                        f"aggregation={INV_AGG_REGISTRY[metric_agg]}"
646
                    )
647
                    self._aggregation_list[metric_name] = metric_agg
lintangsutawika's avatar
lintangsutawika committed
648

649
650
651
652
653
654
                if "higher_is_better" in metric_config:
                    self._higher_is_better[metric_name] = metric_config[
                        "higher_is_better"
                    ]
                else:
                    eval_logger.warning(
655
                        f"[Task: {self.config.task}] metric {metric_name} is defined, but higher_is_better is not. "
656
                        f"using default "
haileyschoelkopf's avatar
haileyschoelkopf committed
657
                        f"higher_is_better={is_higher_better(metric_name)}"
658
                    )
haileyschoelkopf's avatar
haileyschoelkopf committed
659
                    self._higher_is_better[metric_name] = is_higher_better(metric_name)
660

661
        self.download(self.config.dataset_kwargs)
662
663
664
        self._training_docs = None
        self._fewshot_docs = None

665
        if self.config.filter_list is not None:
lintangsutawika's avatar
lintangsutawika committed
666
            self._filters = []
667
            for filter_config in self.config.filter_list:
668
669
670
671
672
673
674
675
676
                filter_name = filter_config["name"]
                filter_functions = filter_config["filter"]
                components = []
                for function in filter_functions:
                    kwargs = {
                        key: function[key] for key in function if key != "function"
                    }
                    components.append([function["function"], kwargs])
                filter_pipeline = build_filter_ensemble(filter_name, components)
lintangsutawika's avatar
lintangsutawika committed
677
                self._filters.append(filter_pipeline)
lintangsutawika's avatar
lintangsutawika committed
678
        else:
679
            self._filters = [build_filter_ensemble("none", [["take_first", None]])]
680

681
682
        if self.config.use_prompt is not None:
            eval_logger.info(f"loading prompt {self.config.use_prompt}")
683
            self.prompt = get_prompt(
684
                self.config.use_prompt, self.DATASET_PATH, self.DATASET_NAME
lintangsutawika's avatar
lintangsutawika committed
685
            )
686
687
688
        else:
            self.prompt = None

lintangsutawika's avatar
lintangsutawika committed
689
        if self.fewshot_docs() is not None:
haileyschoelkopf's avatar
haileyschoelkopf committed
690
            self.sampler = samplers.get_sampler(
haileyschoelkopf's avatar
haileyschoelkopf committed
691
692
693
                self.config.fewshot_config.get("sampler", "default")
                if self.config.fewshot_config
                else "default"
haileyschoelkopf's avatar
haileyschoelkopf committed
694
            )(list(self.fewshot_docs()), self, rnd=random.Random(1234))
695

696
        if self.has_test_docs():
697
            self.task_docs = self.test_docs()
698
        elif self.has_validation_docs():
699
            self.task_docs = self.validation_docs()
700
        else:
701
            assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!"
702

703
        # Test One Doc
704
        self.features = list(self.task_docs.features.keys())
705
706
        self.multiple_input = 0
        self.multiple_target = 0
707
        test_doc = self.task_docs[0]
708
        test_text = self.doc_to_text(test_doc)
709
        test_target = self.doc_to_target(test_doc)
lintangsutawika's avatar
lintangsutawika committed
710

711
        if self.config.doc_to_choice is not None:
lintangsutawika's avatar
lintangsutawika committed
712
            test_choice = self.doc_to_choice(test_doc)
713
            if not isinstance(test_choice, list):
lintangsutawika's avatar
lintangsutawika committed
714
                eval_logger.error("doc_to_choice must return list")
715
716
            else:
                num_choice = len(test_choice)
717

718
            if isinstance(test_text, int):
719
                self.multiple_input = num_choice
720
721
        else:
            test_choice = None
722

723
        if isinstance(test_target, list):
724
            self.multiple_target = len(test_target)
725
        else:
726
            if (isinstance(test_target, int)) and (test_choice is not None):
lintangsutawika's avatar
lintangsutawika committed
727
                test_target = test_choice[test_target]
728
            else:
lintangsutawika's avatar
lintangsutawika committed
729
                test_target = str(test_target)
730

731
732
733
        if test_choice is not None:
            check_choices = test_choice
        else:
lintangsutawika's avatar
lintangsutawika committed
734
            check_choices = [test_target]
735
736
737
738
        if self.config.doc_to_choice is not None:
            for choice in check_choices:
                choice_has_whitespace = True if choice[0].isspace() else False
                delimiter_has_whitespace = (
739
740
                    True
                    if self.config.target_delimiter.rstrip()
741
                    != self.config.target_delimiter
742
                    else False
743
                )
744

745
                if delimiter_has_whitespace and choice_has_whitespace:
746
747
                    eval_logger.debug(
                        f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" have whitespace'
748
749
                    )
                elif (not delimiter_has_whitespace) and (not choice_has_whitespace):
750
                    eval_logger.debug(
751
                        f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace'
752
753
                    )

Ethan Smith's avatar
Ethan Smith committed
754
    def download(self, dataset_kwargs=None) -> None:
755
756
757
758
759
760
        self.dataset = datasets.load_dataset(
            path=self.DATASET_PATH,
            name=self.DATASET_NAME,
            **dataset_kwargs if dataset_kwargs is not None else {},
        )

baberabb's avatar
baberabb committed
761
    def has_training_docs(self) -> bool:
762
        if self.config.training_split is not None:
763
764
765
766
            return True
        else:
            return False

baberabb's avatar
baberabb committed
767
    def has_validation_docs(self) -> bool:
768
        if self.config.validation_split is not None:
769
770
771
772
            return True
        else:
            return False

baberabb's avatar
baberabb committed
773
    def has_test_docs(self) -> bool:
774
        if self.config.test_split is not None:
775
776
777
778
            return True
        else:
            return False

baberabb's avatar
baberabb committed
779
    def training_docs(self) -> datasets.Dataset:
780
        if self.has_training_docs():
781
782
783
            if self.config.process_docs is not None:
                return self.config.process_docs(
                    self.dataset[self.config.training_split]
784
                )
785
            return self.dataset[self.config.training_split]
786

baberabb's avatar
baberabb committed
787
    def validation_docs(self) -> datasets.Dataset:
788
        if self.has_validation_docs():
789
790
791
            if self.config.process_docs is not None:
                return self.config.process_docs(
                    self.dataset[self.config.validation_split]
792
                )
793
            return self.dataset[self.config.validation_split]
794

baberabb's avatar
baberabb committed
795
    def test_docs(self) -> datasets.Dataset:
796
        if self.has_test_docs():
797
798
799
            if self.config.process_docs is not None:
                return self.config.process_docs(self.dataset[self.config.test_split])
            return self.dataset[self.config.test_split]
800

801
    def fewshot_docs(self):
802
        if self.config.fewshot_split is not None:
803
804
            if self.config.process_docs is not None:
                return self.config.process_docs(self.dataset[self.config.fewshot_split])
805
            return self.dataset[self.config.fewshot_split]
806
        else:
807
            if (self.config.num_fewshot is not None) and (self.config.num_fewshot > 0):
808
                eval_logger.warning(
809
                    f"Task '{self.config.task}': "
810
811
812
813
                    "num_fewshot > 0 but fewshot_split is None. "
                    "using preconfigured rule."
                )
            return super().fewshot_docs()
814

lintangsutawika's avatar
lintangsutawika committed
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
    @utils.positional_deprecated
    def fewshot_context(self, doc, num_fewshot):
        """Returns a fewshot context string that is made up of a prepended description
        (if provided), the `num_fewshot` number of examples, and an appended prompt example.

        :param doc: str
            The document as returned from training_docs, validation_docs, or test_docs.
        :param num_fewshot: int
            The number of fewshot examples to provide in the returned context string.
        :returns: str
            The fewshot context.
        """

        if num_fewshot == 0:
            # always prepend the (possibly empty) task description
            labeled_examples = self.config.description
        else:
            labeled_examples = self.config.description + self.sampler.get_context(
                doc, num_fewshot
            )

        example = self.doc_to_text(doc)
837
838
839
840
841
842
843
844
845
846
847
848
849
        if self.multiple_input:
            return labeled_examples
        else:
            if isinstance(example, str):
                return labeled_examples + example
            elif isinstance(example, list):
                return [labeled_examples + ex for ex in example]
            elif isinstance(example, int):
                if self.config.doc_to_choice is not None:
                    choices = self.doc_to_choice(doc)
                    return labeled_examples + choices[example]
                else:
                    return labeled_examples + str(example)
lintangsutawika's avatar
lintangsutawika committed
850

851
852
853
    def apply_filters(self):
        if hasattr(self, "_filters"):
            for f in self._filters:
854
                f.apply(self._instances)
855
856
857
858
        else:
            eval_logger.warning("No filter defined, passing through instances")
            return self._instances

859
    def should_decontaminate(self):
860
        return self.config.should_decontaminate
861
862

    def doc_to_decontamination_query(self, doc):
863
        if self.config.should_decontaminate:
864
865
            if self.config.doc_to_decontamination_query is None:
                return self.doc_to_text(doc)
866
            else:
867
868
869
870
871
872
873
874
875
876
877
                doc_to_decontamination_query = self.config.doc_to_decontamination_query
                if doc_to_decontamination_query in self.features:
                    return doc[doc_to_decontamination_query]
                elif callable(doc_to_decontamination_query):
                    return doc_to_decontamination_query(doc)
                else:
                    return ast.literal_eval(
                        utils.apply_template(
                            self.config.doc_to_decontamination_query, doc
                        )
                    )
878

879
880
881
882
883
884
885
886
887
888
889
890
    def _process_doc(self, doc):
        """
        Override this to process (detokenize, strip, replace, etc.) individual
        documents. This can be used in a map over documents of a data split.
        E.g. `map(self._process_doc, self.dataset["validation"])`

        :return: dict
            The processed version of the specified `doc`.
        """
        return doc

    def doc_to_text(self, doc):
891
892
        if self.prompt is not None:
            doc_to_text = self.prompt
893
        else:
894
            doc_to_text = self.config.doc_to_text
lintangsutawika's avatar
lintangsutawika committed
895

896
        if isinstance(doc_to_text, int):
897
            return doc_to_text
898
        elif isinstance(doc_to_text, str):
899
            if doc_to_text in self.features:
900
                # if self.config.doc_to_choice is not None:
901
902
                #     return self.doc_to_choice(doc)[doc[doc_to_text]]
                # else:
903
904
                return doc[doc_to_text]
            else:
lintangsutawika's avatar
lintangsutawika committed
905
                text_string = utils.apply_template(doc_to_text, doc)
lintangsutawika's avatar
lintangsutawika committed
906
                if text_string.isdigit() and self._config.doc_to_choice is not None:
lintangsutawika's avatar
lintangsutawika committed
907
908
909
                    return ast.literal_eval(text_string)
                else:
                    return text_string
910
        elif callable(doc_to_text):
911
            return doc_to_text(doc)
lintangsutawika's avatar
lintangsutawika committed
912
        # Used when applying a Promptsource template
913
        elif hasattr(doc_to_text, "apply"):
914
915
916
917
918
            applied_prompt = doc_to_text.apply(doc)
            if len(applied_prompt) == 2:
                return applied_prompt[0]
            else:
                eval_logger.warning("Applied prompt returns empty string")
919
                return self.config.fewshot_delimiter
920
        else:
921
            print(type(doc_to_text))
922
            raise TypeError
923

924
    def doc_to_target(self, doc: dict) -> Union[int, str, list]:
925
926
        if self.prompt is not None:
            doc_to_target = self.prompt
927
        else:
928
            doc_to_target = self.config.doc_to_target
929

930
        if isinstance(doc_to_target, int):
931
            return doc_to_target
932
        elif isinstance(doc_to_target, str):
933
            if doc_to_target in self.features:
934
                # if self.config.doc_to_choice is not None:
935
936
937
                #     return self.doc_to_choice(doc)[doc[doc_to_target]]
                # else:
                return doc[doc_to_target]
938
            else:
lintangsutawika's avatar
lintangsutawika committed
939
                target_string = utils.apply_template(doc_to_target, doc)
lintangsutawika's avatar
lintangsutawika committed
940
                if target_string.isdigit() and self._config.doc_to_choice is not None:
lintangsutawika's avatar
lintangsutawika committed
941
                    return ast.literal_eval(target_string)
lintangsutawika's avatar
lintangsutawika committed
942
943
944
945
946
                elif (
                    len(target_string) >= 2
                    and (target_string[0] == "[")
                    and (target_string[-1] == "]")
                ):
baberabb's avatar
baberabb committed
947
948
949
950
                    try:
                        return ast.literal_eval(target_string)
                    except (SyntaxError, ValueError):
                        return target_string
lintangsutawika's avatar
lintangsutawika committed
951
952
                else:
                    return target_string
953
        elif isinstance(doc_to_target, list):
954
            return doc_to_target
955
        elif callable(doc_to_target):
956
            return doc_to_target(doc)
lintangsutawika's avatar
lintangsutawika committed
957
        # Used when applying a Promptsource template
958
        elif hasattr(doc_to_target, "apply"):
959
            applied_prompt = doc_to_target.apply(doc)
960
961
962
963
            if len(applied_prompt) == 2:
                return applied_prompt[1]
            else:
                eval_logger.warning("Applied prompt returns empty string")
964
                return self.config.fewshot_delimiter
965
966
        else:
            raise TypeError
967

baberabb's avatar
baberabb committed
968
    def doc_to_choice(self, doc: Any) -> List[str]:
969
970
        if self.prompt is not None:
            doc_to_choice = self.prompt
971
        elif self.config.doc_to_choice is None:
972
973
            eval_logger.error("doc_to_choice was called but not set in config")
        else:
974
            doc_to_choice = self.config.doc_to_choice
975

976
        if isinstance(doc_to_choice, str):
977
978
979
980
            if doc_to_choice in self.features:
                return doc[doc_to_choice]
            else:
                return ast.literal_eval(utils.apply_template(doc_to_choice, doc))
981
        elif isinstance(doc_to_choice, list):
982
            return doc_to_choice
983
        elif isinstance(doc_to_choice, dict):
984
985
986
987
988
989
990
            return list(doc_to_choice.values())
        elif callable(doc_to_choice):
            return doc_to_choice(doc)
        elif hasattr(doc_to_choice, "get_answer_choices_list"):
            return doc_to_choice.get_answer_choices_list(doc)
        else:
            raise TypeError
991

baberabb's avatar
baberabb committed
992
993
994
    def construct_requests(
        self, doc: dict, ctx: str, **kwargs
    ) -> Union[List[Instance], Instance]:
995
        if self.OUTPUT_TYPE == "loglikelihood":
lintangsutawika's avatar
lintangsutawika committed
996
            arguments = (ctx, self.doc_to_target(doc))
997
        elif self.OUTPUT_TYPE == "loglikelihood_rolling":
lintangsutawika's avatar
lintangsutawika committed
998
            arguments = (self.doc_to_target(doc),)
999
        elif self.OUTPUT_TYPE == "multiple_choice":
1000
            choices = self.doc_to_choice(doc)
1001
            target_delimiter = self.config.target_delimiter
1002
1003
            if self.multiple_input:
                # If there are multiple inputs, choices are placed in the ctx
1004
                cont = self.doc_to_target(doc)
1005
1006
1007
                arguments = [
                    (ctx + choice, f"{target_delimiter}{cont}") for choice in choices
                ]
1008
            else:
1009
                # Otherwise they are placed in the continuation
1010
                arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices]
1011

1012
            request_list = [
1013
1014
                Instance(
                    request_type="loglikelihood",
lintangsutawika's avatar
lintangsutawika committed
1015
                    doc=doc,
1016
                    arguments=arg,
1017
                    idx=i,
1018
1019
                    **kwargs,
                )
1020
                for i, arg in enumerate(arguments)
1021
            ]
1022
            # TODO: we should raise a warning telling users this will at most ~2x runtime.
1023
            if "acc_mutual_info" in self._metric_fn_list.keys():
1024
1025
1026
                # if we are calculating multiple choice accuracy
                # using mutual information instead of raw loglikelihood as metric, need unconditional lls.

lintangsutawika's avatar
lintangsutawika committed
1027
                # here mutual info refers to calculating
1028
1029
1030
1031
1032
1033
                # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice))
                # in other words normalizing by subtracting the unconditional logprob of each choice.
                request_list.extend(
                    [
                        Instance(
                            request_type="loglikelihood",
lintangsutawika's avatar
lintangsutawika committed
1034
                            doc=doc,
1035
                            arguments=("", "{}".format(choice)),
1036
1037
1038
                            idx=i,
                            **kwargs,
                        )
lintangsutawika's avatar
lintangsutawika committed
1039
                        for i, choice in enumerate(choices)
1040
1041
1042
                    ]
                )
            return request_list
lintangsutawika's avatar
lintangsutawika committed
1043

1044
        elif self.OUTPUT_TYPE == "generate_until":
1045
            arguments = (ctx, self.config.generation_kwargs)
lintangsutawika's avatar
lintangsutawika committed
1046
1047

        return Instance(
lintangsutawika's avatar
lintangsutawika committed
1048
1049
            request_type=self.OUTPUT_TYPE, doc=doc, arguments=arguments, idx=0, **kwargs
        )
1050
1051

    def process_results(self, doc, results):
1052
        if callable(self.config.process_results):
Lintang Sutawika's avatar
Lintang Sutawika committed
1053
            return self.config.process_results(doc, results)
lintangsutawika's avatar
lintangsutawika committed
1054

1055
        result_dict = {}
1056
        use_metric = list(self._metric_fn_list.keys())
1057
1058
1059
        if self.OUTPUT_TYPE == "loglikelihood":
            results = results[0]
            ll, is_greedy = results
1060
1061
            prob_norm = np.exp(ll)

1062
1063
1064
            return {
                **({"perplexity": ll} if "perplexity" in use_metric else {}),
                **({"acc": int(is_greedy)} if "acc" in use_metric else {}),
1065
                **(
lintangsutawika's avatar
lintangsutawika committed
1066
                    {"brier_score": (0, [prob_norm])}  # Gold is Index 0
1067
1068
1069
                    if "brier_score" in use_metric
                    else {}
                ),
1070
            }
1071
        elif self.OUTPUT_TYPE == "loglikelihood_rolling":
haileyschoelkopf's avatar
haileyschoelkopf committed
1072
            (loglikelihood,) = results
1073
1074
            _words = self.count_words(self.doc_to_target(doc))
            _bytes = self.count_bytes(self.doc_to_target(doc))
haileyschoelkopf's avatar
haileyschoelkopf committed
1075
            return {
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
                **(
                    {"word_perplexity": (loglikelihood, _words)}
                    if "word_perplexity" in use_metric
                    else {}
                ),
                **(
                    {"byte_perplexity": (loglikelihood, _bytes)}
                    if "byte_perplexity" in use_metric
                    else {}
                ),
                **(
                    {"bits_per_byte": (loglikelihood, _bytes)}
                    if "bits_per_byte" in use_metric
                    else {}
                ),
haileyschoelkopf's avatar
haileyschoelkopf committed
1091
            }
1092
        elif self.OUTPUT_TYPE == "multiple_choice":
1093
            lls, is_greedy = zip(*results)
lintangsutawika's avatar
lintangsutawika committed
1094

1095
            # retrieve choices in List[str] form, to compute choice lengths, etc.
1096
            choices = self.doc_to_choice(doc)
1097
1098
            completion_len = np.array([float(len(i)) for i in choices])

1099
1100
            if (
                2 * len(choices) == len(lls)
1101
                and "acc_mutual_info" in self._metric_fn_list.keys()
1102
1103
1104
1105
1106
1107
1108
            ):
                # then we are doing mutual info.
                # this stores the "dryrun" / unconditional answer loglikelihoods
                lls_unconditional = lls[1::2]
                assert len(lls_unconditional) == len(choices)
                # and this stores our "regular" conditional loglikelihoods
                lls = lls[::2]
1109

1110
1111
            pred = np.argmax(lls)
            pred_norm = np.argmax(lls / completion_len)
lintangsutawika's avatar
lintangsutawika committed
1112

1113
1114
            if self.multiple_input:
                gold = self.doc_to_text(doc)
1115
            else:
1116
                gold = self.doc_to_target(doc)
1117
1118

            gold_index_error = False
1119
            if isinstance(gold, list):
Lintang Sutawika's avatar
Lintang Sutawika committed
1120
1121
                gold = [i if i < len(choices) else -100 for i in gold]
                if -100 in gold:
1122
1123
                    gold_index_error = True
            else:
1124
                if isinstance(gold, int):
Lintang Sutawika's avatar
Lintang Sutawika committed
1125
                    gold = gold if gold < len(choices) else -100
1126
                elif isinstance(gold, str):
Lintang Sutawika's avatar
Lintang Sutawika committed
1127
                    gold = choices.index(gold) if gold in choices else -100
lintangsutawika's avatar
lintangsutawika committed
1128

Lintang Sutawika's avatar
Lintang Sutawika committed
1129
                if gold == -100:
1130
1131
1132
1133
                    gold_index_error = True

            if gold_index_error:
                eval_logger.warning(
lintangsutawika's avatar
lintangsutawika committed
1134
                    f"Label index was not in within range of available choices,"
1135
1136
                    f"Sample:\n\n{doc}\n\n"
                )
lintangsutawika's avatar
lintangsutawika committed
1137

1138
            if self.multiple_target:
lintangsutawika's avatar
lintangsutawika committed
1139
1140
                acc = 1.0 if pred in gold else 0.0
                acc_norm = 1.0 if pred_norm in gold else 0.0
Lintang Sutawika's avatar
Lintang Sutawika committed
1141
                exact_match = int(any([is_greedy[i] if i != -100 else 0 for i in gold]))
lintangsutawika's avatar
lintangsutawika committed
1142
1143
1144
            else:
                acc = 1.0 if pred == gold else 0.0
                acc_norm = 1.0 if pred_norm == gold else 0.0
1145
                # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly
Lintang Sutawika's avatar
Lintang Sutawika committed
1146
                exact_match = int(is_greedy[gold]) if gold != -100 else 0
1147

lintangsutawika's avatar
lintangsutawika committed
1148
            prob_norm = utils.softmax(lls)
lintangsutawika's avatar
lintangsutawika committed
1149

lintangsutawika's avatar
lintangsutawika committed
1150
            # TODO use keyword arguments to the metric?
lintangsutawika's avatar
format  
lintangsutawika committed
1151
            # gold, pred, norm stuff, the original lls,
1152
            result_dict = {
1153
                **({"acc": acc} if "acc" in use_metric else {}),
1154
1155
                **({"f1": (gold, pred)} if "f1" in use_metric else {}),
                **({"mcc": (gold, pred)} if "mcc" in use_metric else {}),
1156
                **({"acc_norm": acc_norm} if "acc_norm" in use_metric else {}),
1157
                **({"exact_match": exact_match} if "exact_match" in use_metric else {}),
lintangsutawika's avatar
format  
lintangsutawika committed
1158
                **(
1159
1160
                    # {"brier_score": (gold, prob_norm)}
                    {"brier_score": [np.eye(len(prob_norm))[gold], prob_norm]}
lintangsutawika's avatar
format  
lintangsutawika committed
1161
1162
1163
                    if "brier_score" in use_metric
                    else {}
                ),
1164
1165
            }

1166
            if "acc_mutual_info" in use_metric:
lintangsutawika's avatar
lintangsutawika committed
1167
1168
1169
                lls_mutual_info = [
                    ll_c - ll_u for ll_c, ll_u in zip(lls, lls_unconditional)
                ]
1170
1171
1172
                acc_mutual_info = 1.0 if np.argmax(lls_mutual_info) == gold else 0.0
                result_dict["acc_mutual_info"] = acc_mutual_info

1173
        elif self.OUTPUT_TYPE == "generate_until":
1174
            gold = self.doc_to_target(doc)
Chris's avatar
Chris committed
1175
            result = results[0]
1176
            if self.config.doc_to_choice is not None:
lintangsutawika's avatar
lintangsutawika committed
1177
                # If you set doc_to_choice,
lintangsutawika's avatar
lintangsutawika committed
1178
                # it assumes that doc_to_target returns a number.
1179
1180
                choices = self.doc_to_choice(doc)
                gold = choices[gold]
1181
1182
            # we expect multiple_targets to be a list.
            elif self.multiple_target:
baberabb's avatar
baberabb committed
1183
                gold = list(gold)
Chris's avatar
Chris committed
1184
1185
1186
            elif type(gold) != type(result):
                # cast gold to the same type as result
                gold = type(result)(gold)
1187

lintangsutawika's avatar
lintangsutawika committed
1188
            for metric in self._metric_fn_list.keys():
haileyschoelkopf's avatar
haileyschoelkopf committed
1189
1190
1191
1192
1193
                if self.multiple_target:
                    # in the case where we have multiple targets,
                    # return true if any are true
                    # TODO: this may break for multipLe_target, non zero-or-1 metrics
                    scores = []
haileyschoelkopf's avatar
haileyschoelkopf committed
1194
1195
1196
1197
                    if not isinstance(gold, list):
                        # sometimes, a multiple_target dataset has exceptions where one doc has only one string answer
                        # print(gold)
                        gold = [gold]
1198
1199
1200
1201
1202
1203
1204
1205
                    if metric == "exact_match":
                        result = [result for _ in range(len(gold))]
                        scores = self._metric_fn_list[metric](
                            references=gold,
                            predictions=result,
                            **self._metric_fn_kwargs[metric],
                        )[metric]
                        result_score = 1.0 if scores > 0.0 else 0.0
haileyschoelkopf's avatar
haileyschoelkopf committed
1206
                    else:
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
                        for gold_option in gold:
                            try:
                                result_score = self._metric_fn_list[metric](
                                    references=[gold_option],
                                    predictions=[result],
                                    **self._metric_fn_kwargs[metric],
                                )
                            except (
                                TypeError
                            ):  # TODO: this is hacky and I don't want to do it
                                result_score = self._metric_fn_list[metric](
                                    [gold_option, result]
                                )
                            if isinstance(result_score, dict):
                                # TODO: this handles the case where HF evaluate returns a dict.
                                result_score = result_score[metric]
                            scores.append(result_score)
                        if any(scores):
                            result_score = 1.0
                        else:
                            result_score = 0.0
haileyschoelkopf's avatar
haileyschoelkopf committed
1228
                else:
1229
                    try:
1230
                        result_score = self._metric_fn_list[metric](
1231
1232
                            references=[gold],
                            predictions=[result],
1233
                            **self._metric_fn_kwargs[metric],
1234
                        )
1235
                    except TypeError:  # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics
1236
                        result_score = self._metric_fn_list[metric]([gold, result])
1237
1238
1239
1240
                    if isinstance(result_score, dict):
                        # TODO: this handles the case where HF evaluate returns a dict.
                        result_score = result_score[metric]
                result_dict[metric] = result_score
1241
        else:
lintangsutawika's avatar
lintangsutawika committed
1242
1243
            raise ValueError(
                f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ",
1244
                "'loglikelihood', 'loglikelihood_rolling', 'generate_until' or 'multiple_choice'",
1245
            )
1246
1247
1248

        return result_dict

Baber Abbasi's avatar
Baber Abbasi committed
1249
    def aggregation(self) -> dict:
1250
1251
        return self._aggregation_list

Baber Abbasi's avatar
Baber Abbasi committed
1252
    def higher_is_better(self) -> dict:
haileyschoelkopf's avatar
haileyschoelkopf committed
1253
        return self._higher_is_better
1254

Baber Abbasi's avatar
Baber Abbasi committed
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
    def get_config(self, key: str) -> Any:
        return getattr(self._config, key, None)

    def override_metric(self, metric_name: str) -> None:
        """
        Override the default metrics used for evaluation with custom metrics.

        Parameters:
        - metric_name (str): The name of the custom metric to override. Should be registered in api.metrics.
        """
        (
            self._metric_fn_list,
            self._aggregation_list,
            self._metric_fn_kwargs,
            self._higher_is_better,
        ) = ({}, {}, {}, {})
        self._metric_fn_list[metric_name] = get_metric(metric_name)
        self._aggregation_list[metric_name] = get_metric_aggregation(metric_name)
        self._higher_is_better[metric_name] = is_higher_better(metric_name)
        self._metric_fn_kwargs[metric_name] = {}
        setattr(self._config, "metric_list", [{"metric": metric_name}])
        setattr(self._config, "process_results", None)

    def override_config(
        self, key: str = None, value: Any = None, update: bool = False
    ) -> None:
        if update:
            current_value = getattr(self._config, key)
            assert isinstance(current_value, dict)
            current_value.update(value)
            setattr(self._config, key, current_value)
        else:
            setattr(self._config, key, value)

1289
1290
1291
1292

class MultipleChoiceTask(Task):
    OUTPUT_TYPE: str = "loglikelihood"

baberabb's avatar
baberabb committed
1293
    def doc_to_target(self, doc: dict) -> str:
1294
1295
        return " " + doc["choices"][doc["gold"]]

baberabb's avatar
baberabb committed
1296
    def construct_requests(self, doc: dict, ctx: str, **kwargs) -> List[Instance]:
1297
        # TODO: add mutual info here?
lintangsutawika's avatar
lintangsutawika committed
1298
1299
        return [
            Instance(
haileyschoelkopf's avatar
haileyschoelkopf committed
1300
                request_type="loglikelihood",
lintangsutawika's avatar
lintangsutawika committed
1301
                doc=doc,
1302
                arguments=(ctx, " {}".format(choice)),
1303
                idx=i,
1304
1305
                **kwargs,
            )
lintangsutawika's avatar
lintangsutawika committed
1306
1307
            for i, choice in enumerate(doc["choices"])
        ]
1308

baberabb's avatar
baberabb committed
1309
    def process_results(self, doc: dict, results: List[Tuple[float, bool]]) -> dict:
lintangsutawika's avatar
lintangsutawika committed
1310
1311
1312
        results = [
            res[0] for res in results
        ]  # only retain loglikelihoods, discard is_greedy TODO: do we need is_greedy anywhere?
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
        gold = doc["gold"]

        acc = 1.0 if np.argmax(results) == gold else 0.0
        completion_len = np.array([float(len(i)) for i in doc["choices"]])
        acc_norm = 1.0 if np.argmax(results / completion_len) == gold else 0.0

        return {
            "acc": acc,
            "acc_norm": acc_norm,
        }

baberabb's avatar
baberabb committed
1324
    def higher_is_better(self) -> dict:
1325
1326
1327
1328
1329
        return {
            "acc": True,
            "acc_norm": True,
        }

baberabb's avatar
baberabb committed
1330
    def aggregation(self) -> dict:
1331
1332
1333
1334
1335
1336
        return {
            "acc": mean,
            "acc_norm": mean,
        }


lintangsutawika's avatar
lintangsutawika committed
1337
class PerplexityTask(Task):
1338
1339
    OUTPUT_TYPE = "loglikelihood_rolling"

baberabb's avatar
baberabb committed
1340
    def has_training_docs(self) -> bool:
1341
1342
        return False

baberabb's avatar
baberabb committed
1343
    def fewshot_examples(self, k: int, rnd) -> List:
1344
1345
1346
        assert k == 0
        return []

baberabb's avatar
baberabb committed
1347
    def fewshot_context(self, doc: dict, num_fewshot: int) -> Literal[""]:
1348
1349
1350
1351
1352
1353
        assert (
            num_fewshot == 0
        ), "The number of fewshot examples must be 0 for perplexity tasks."

        return ""

baberabb's avatar
baberabb committed
1354
    def higher_is_better(self) -> dict:
1355
1356
1357
1358
1359
1360
1361
1362
1363
        return {
            "word_perplexity": False,
            "byte_perplexity": False,
            "bits_per_byte": False,
        }

    def doc_to_decontamination_query(self, doc):
        return doc

Ethan Smith's avatar
Ethan Smith committed
1364
    def doc_to_text(self, doc) -> str:
1365
1366
1367
1368
1369
        return ""

    def doc_to_target(self, doc):
        return doc

baberabb's avatar
baberabb committed
1370
    def construct_requests(self, doc: dict, ctx: Union[str, None], **kwargs):
1371
1372
        assert not ctx

lintangsutawika's avatar
lintangsutawika committed
1373
1374
1375
1376
1377
1378
1379
        return Instance(
            request_type=self.OUTPUT_TYPE,
            doc=doc,
            arguments=(self.doc_to_target(doc),),
            idx=0,
            **kwargs,
        )
1380

baberabb's avatar
baberabb committed
1381
    def process_results(self, doc: dict, results: float) -> dict:
1382
        (loglikelihood,) = results
haileyschoelkopf's avatar
haileyschoelkopf committed
1383
1384
        words = self.count_words(self.doc_to_target(doc))
        bytes_ = self.count_bytes(self.doc_to_target(doc))
1385
1386
1387
1388
1389
1390
        return {
            "word_perplexity": (loglikelihood, words),
            "byte_perplexity": (loglikelihood, bytes_),
            "bits_per_byte": (loglikelihood, bytes_),
        }

baberabb's avatar
baberabb committed
1391
    def aggregation(self) -> dict:
1392
1393
1394
1395
1396
1397
1398
        return {
            "word_perplexity": weighted_perplexity,
            "byte_perplexity": weighted_perplexity,
            "bits_per_byte": bits_per_byte,
        }

    @classmethod
baberabb's avatar
baberabb committed
1399
    def count_bytes(cls, doc) -> int:
1400
1401
1402
        return len(doc.encode("utf-8"))

    @classmethod
baberabb's avatar
baberabb committed
1403
    def count_words(cls, doc) -> int:
1404
1405
        """Downstream tasks with custom word boundaries should override this!"""
        return len(re.split(r"\s+", doc))