Merge branch 'main' into inverse-scaling-tasks

lm_eval/tasks/scrolls/scrolls_contractnli.yaml

+group: scrolls
+task: scrolls_contractnli
+class: !function task.ContractNLI

lm_eval/tasks/scrolls/scrolls_govreport.yaml

+group: scrolls
+task: scrolls_govreport
+class: !function task.GovReport

lm_eval/tasks/scrolls/scrolls_narrativeqa.yaml

+group: scrolls
+task: scrolls_narrativeqa
+class: !function task.NarrativeQA

lm_eval/tasks/scrolls/scrolls_qasper.yaml

+group: scrolls
+task: scrolls_qasper
+class: !function task.Qasper

lm_eval/tasks/scrolls/scrolls_qmsum.yaml

+group: scrolls
+task: scrolls_qmsum
+class: !function task.QMSum

lm_eval/tasks/scrolls/scrolls_quality.yaml

+group: scrolls
+task: scrolls_quality
+class: !function task.QuALITY

lm_eval/tasks/scrolls/scrolls_summscreenfd.yaml

+group: scrolls
+task: scrolls_summscreenfd
+class: !function task.SummScreenFD

lm_eval/tasks/squad_completion/README.md

+# Squad-completion
+
+### Paper
+
+Title: Simple Linear Attention Language Models Balance The Recall-Throughput Tradeoff
+
+A Variant of the SQuAD question answering task, as implemented by Based. See [https://github.com/EleutherAI/lm-evaluation-harness/lm_eval/tasks/squadv2/README.md] for more info.
+
+Homepage: https://github.com/HazyResearch/based-evaluation-harness
+
+
+
+
+### Citation
+
+```
+@misc{arora2024simple,
+      title={Simple linear attention language models balance the recall-throughput tradeoff},
+      author={Simran Arora and Sabri Eyuboglu and Michael Zhang and Aman Timalsina and Silas Alberti and Dylan Zinsley and James Zou and Atri Rudra and Christopher Ré},
+      year={2024},
+      eprint={2402.18668},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+
+@misc{rajpurkar2018know,
+    title={Know What You Don't Know: Unanswerable Questions for SQuAD},
+    author={Pranav Rajpurkar and Robin Jia and Percy Liang},
+    year={2018},
+    eprint={1806.03822},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+
+```
+
+### Groups and Tasks
+
+#### Tasks
+
+* `squad_completion`: the SQuAD task as implemented in the paper "Simple linear attention language models balance the recall-throughput tradeoff". Designed for zero-shot evaluation of small LMs.
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/squad_completion/squad_completion.yaml

+task: squad_completion
+class: !function task.SQUADCompletion

lm_eval/tasks/squad_completion/task.py

+"""
+"""
+import re
+from typing import List
+
+import numpy as np
+
+from lm_eval.api.instance import Instance
+from lm_eval.api.task import ConfigurableTask
+
+
+class SQUADCompletion(ConfigurableTask):
+    VERSION = 0
+    DATASET_PATH = "hazyresearch/based-squad"
+    DATASET_NAME = "default"
+
+    def __init__(self):
+        super().__init__(config={"metadata": {"version": self.VERSION}})
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False
+
+    def validation_docs(self):
+        return self.dataset["validation"]
+
+    def doc_to_text(self, doc):
+        return doc["text"]
+
+    def doc_to_target(self, doc):
+        return doc["value"]
+
+    def construct_requests(self, doc, ctx, **kwargs):
+        """Uses RequestFactory to construct Requests and returns an iterable of
+        Requests which will be sent to the LM.
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param ctx: str
+            The context string, generated by fewshot_context. This includes the natural
+            language description, as well as the few shot examples, and the question
+            part of the document for `doc`.
+        """
+
+        return [
+            Instance(
+                request_type="generate_until",
+                doc=doc,
+                arguments=(ctx, {"until": ["\n"], "max_gen_toks": 48}),
+                idx=0,
+                **kwargs,
+            )
+        ]
+
+    def process_results(self, doc, results):
+        """Take a single document and the LM results and evaluates, returning a
+        dict where keys are the names of submetrics and values are the values of
+        the metric for that one document
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param results:
+            The results of the requests created in construct_requests.
+        """
+        # continuation, (logprob_unanswerable, _) = results
+        continuation = results
+
+        return {"contains": contains_score(continuation[0], [doc["value"]])}
+
+    def aggregation(self):
+        """
+        :returns: {str: [float] -> float}
+            A dictionary where keys are the names of submetrics and values are
+            functions that aggregate a list of metrics
+        """
+        return {
+            "contains": np.mean,  # Exact match (the normalized answer exactly match the gold answer)
+        }
+
+    def higher_is_better(self):
+        """
+        :returns: {str: bool}
+            A dictionary where keys are the names of submetrics and values are
+            whether a higher value of the submetric is better
+        """
+        return {
+            "contains": True,  # Exact match (the normalized answer exactly match the gold answer
+        }
+
+
+def contains_score(prediction: str, labels: List[str]):
+    return max(
+        int(bool(re.search(re.compile(re.escape(label), re.IGNORECASE), prediction)))
+        for label in labels
+    )

lm_eval/tasks/super_glue/record/default.yaml

@@ -7,8 +7,9 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 doc_to_text: !function util.doc_to_text
-doc_to_target: "{{answers}}"
-doc_to_choice: "{{entities}}"
+doc_to_target: !function util.doc_to_target
+doc_to_choice: !function util.doc_to_choice
+process_docs: !function util.process_docs
 process_results: !function util.process_results
 metric_list:
   - metric: f1
@@ -17,4 +18,4 @@ metric_list:
     higher_is_better: True
     aggregation: mean
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/super_glue/record/util.py

+import datasets
 import numpy as np
 import transformers.data.metrics.squad_metrics as squad_metrics
 
@@ -21,6 +22,22 @@ def doc_to_target(doc):
     return format_answer(query=doc["query"], entity=doc["answers"][0])
 
 
+def doc_to_choice(doc):
+    return [format_answer(query=doc["query"], entity=ans) for ans in doc["entities"]]
+
+
+def process_docs(dataset: datasets.Dataset):
+    def _process_doc(doc):
+        return {
+            "passage": doc["passage"],
+            "query": doc["query"],
+            "entities": sorted(list(set(doc["entities"]))),
+            "answers": sorted(list(set(doc["answers"]))),
+        }
+
+    return dataset.map(_process_doc)
+
+
 def process_results(doc, results):
     # ReCoRD's evaluation is actually deceptively simple:
     # - Pick the maximum likelihood prediction entity

lm_eval/tasks/swde/README.md

+# SWDE
+
+### Paper
+
+Title: Language Models Enable Simple Systems For
+Generating Structured Views Of Heterogenous Data
+Lakes
+
+Abstract: A long standing goal of the data management community is to develop general, automated systems
+that ingest semi-structured documents and output queryable tables without human effort or domain
+specific customization. Given the sheer variety of potential documents, state-of-the art systems make
+simplifying assumptions and use domain specific training. In this work, we ask whether we can
+maintain generality by using large language models (LLMs). LLMs, which are pretrained on broad
+data, can perform diverse downstream tasks simply conditioned on natural language task descriptions.
+We propose and evaluate EVAPORATE, a simple, prototype system powered by LLMs. We identify
+two fundamentally different strategies for implementing this system: prompt the LLM to directly
+extract values from documents or prompt the LLM to synthesize code that performs the extraction.
+Our evaluations show a cost-quality tradeoff between these two approaches. Code synthesis is cheap,
+but far less accurate than directly processing each document with the LLM. To improve quality while
+maintaining low cost, we propose an extended code synthesis implementation, EVAPORATE-CODE+,
+which achieves better quality than direct extraction. Our key insight is to generate many candidate
+functions and ensemble their extractions using weak supervision. EVAPORATE-CODE+ not only
+outperforms the state-of-the art systems, but does so using a sublinear pass over the documents with
+the LLM. This equates to a 110× reduction in the number of tokens the LLM needs to process,
+averaged across 16 real-world evaluation settings of 10k documents each.
+
+
+A task for LMs to perform Information Extraction, as implemented by Based.
+
+Homepage: https://github.com/HazyResearch/based-evaluation-harness
+
+
+Description:
+> SWDE (Information Extraction). The task in the SWDE benchmark is to extract semi-structured relations from raw HTML websites. For example, given an IMBD page for a movie (e.g. Harry Potter and the Sorcerer’s Stone) and a relation key (e.g. release date), the model must extract the correct relation value (e.g. 2001). The SWDE benchmark was originally curated by Lockard et al. for the task of open information extraction from the semi-structured web. Because we are evaluating the zero-shot capabilities of relatively small language models, we adapt the task to make it slightly easier. Our task setup is similar after to that used in Arora et al.
+
+### Citation
+
+```
+@misc{arora2024simple,
+      title={Simple linear attention language models balance the recall-throughput tradeoff},
+      author={Simran Arora and Sabri Eyuboglu and Michael Zhang and Aman Timalsina and Silas Alberti and Dylan Zinsley and James Zou and Atri Rudra and Christopher Ré},
+      year={2024},
+      eprint={2402.18668},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+
+@misc{arora2023language,
+      title={Language Models Enable Simple Systems for Generating Structured Views of Heterogeneous Data Lakes},
+      author={Simran Arora and Brandon Yang and Sabri Eyuboglu and Avanika Narayan and Andrew Hojel and Immanuel Trummer and Christopher Ré},
+      year={2023},
+      eprint={2304.09433},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+
+@inproceedings{lockard-etal-2019-openceres,
+    title = "{O}pen{C}eres: {W}hen Open Information Extraction Meets the Semi-Structured Web",
+    author = "Lockard, Colin  and
+      Shiralkar, Prashant  and
+      Dong, Xin Luna",
+    editor = "Burstein, Jill  and
+      Doran, Christy  and
+      Solorio, Thamar",
+    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
+    month = jun,
+    year = "2019",
+    address = "Minneapolis, Minnesota",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/N19-1309",
+    doi = "10.18653/v1/N19-1309",
+    pages = "3047--3056",
+    abstract = "Open Information Extraction (OpenIE), the problem of harvesting triples from natural language text whose predicate relations are not aligned to any pre-defined ontology, has been a popular subject of research for the last decade. However, this research has largely ignored the vast quantity of facts available in semi-structured webpages. In this paper, we define the problem of OpenIE from semi-structured websites to extract such facts, and present an approach for solving it. We also introduce a labeled evaluation dataset to motivate research in this area. Given a semi-structured website and a set of seed facts for some relations existing on its pages, we employ a semi-supervised label propagation technique to automatically create training data for the relations present on the site. We then use this training data to learn a classifier for relation extraction. Experimental results of this method on our new benchmark dataset obtained a precision of over 70{\%}. A larger scale extraction experiment on 31 websites in the movie vertical resulted in the extraction of over 2 million triples.",
+}
+```
+
+### Groups and Tasks
+
+#### Tasks
+
+* `swde`: the SWDE task as implemented in the paper "Simple linear attention language models balance the recall-throughput tradeoff". Designed for zero-shot evaluation of small LMs.
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/swde/swde.yaml

+task: swde
+class: !function task.SWDE

lm_eval/tasks/swde/task.py

+import re
+from typing import List
+
+import numpy as np
+
+from lm_eval.api.instance import Instance
+from lm_eval.api.task import ConfigurableTask
+
+
+class SWDE(ConfigurableTask):
+    VERSION = 0
+    DATASET_PATH = "hazyresearch/based-swde-v2"
+    DATASET_NAME = "default"
+
+    def __init__(self):
+        super().__init__(config={"metadata": {"version": self.VERSION}})
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False
+
+    def validation_docs(self):
+        return self.dataset["validation"]
+
+    def doc_to_text(self, doc):
+        return doc["text"]
+
+    def doc_to_target(self, doc):
+        return doc["value"]
+
+    def construct_requests(self, doc, ctx, **kwargs):
+        """Uses RequestFactory to construct Requests and returns an iterable of
+        Requests which will be sent to the LM.
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param ctx: str
+            The context string, generated by fewshot_context. This includes the natural
+            language description, as well as the few shot examples, and the question
+            part of the document for `doc`.
+        """
+
+        return [
+            Instance(
+                request_type="generate_until",
+                doc=doc,
+                arguments=(ctx, {"until": ["\n"], "max_gen_toks": 48}),
+                idx=0,
+                **kwargs,
+            )
+        ]
+
+    def process_results(self, doc, results):
+        """Take a single document and the LM results and evaluates, returning a
+        dict where keys are the names of submetrics and values are the values of
+        the metric for that one document
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param results:
+            The results of the requests created in construct_requests.
+        """
+        # continuation, (logprob_unanswerable, _) = results
+        continuation = results
+
+        return {"contains": contains_score(continuation[0], [doc["value"]])}
+
+    def aggregation(self):
+        """
+        :returns: {str: [float] -> float}
+            A dictionary where keys are the names of submetrics and values are
+            functions that aggregate a list of metrics
+        """
+        return {
+            "contains": np.mean,  # Exact match (the normalized answer exactly match the gold answer)
+        }
+
+    def higher_is_better(self):
+        """
+        :returns: {str: bool}
+            A dictionary where keys are the names of submetrics and values are
+            whether a higher value of the submetric is better
+        """
+        return {
+            "contains": True,  # Exact match (the normalized answer exactly match the gold answer
+        }
+
+
+def contains_score(prediction: str, labels: List[str]):
+    return max(
+        int(bool(re.search(re.compile(re.escape(label), re.IGNORECASE), prediction)))
+        for label in labels
+    )

lm_eval/tasks/tinyBenchmarks/README.md

+# tinyBenchmarks
+
+### Paper
+
+Title: `tinyBenchmarks: evaluating LLMs with fewer examples`
+
+Abstract: https://arxiv.org/abs/2402.14992
+
+The versatility of large language models (LLMs) led to the creation of diverse benchmarks that thoroughly test a variety of language models' abilities. These benchmarks consist of tens of thousands of examples making evaluation of LLMs very expensive. In this paper, we investigate strategies to reduce the number of evaluations needed to assess the performance of an LLM on several key benchmarks. For example, we show that to accurately estimate the performance of an LLM on MMLU, a popular multiple-choice QA benchmark consisting of 14K examples, it is sufficient to evaluate this LLM on 100 curated examples. We release evaluation tools and tiny versions of popular benchmarks: Open LLM Leaderboard, MMLU, HELM, and AlpacaEval 2.0. Our empirical analysis demonstrates that these tools and tiny benchmarks are sufficient to reliably and efficiently reproduce the original evaluation results.
+
+Homepage: -
+
+All configs and utils mirror the ones from their original dataset!
+
+### Groups and Tasks
+
+#### Groups
+
+* `tinyBenchmarks`
+
+#### Tasks
+
+* `tinyArc`, `tinyGSM8k`, `tinyHellaswag`, `tinyMMLU`, `tinyTruthfulQA`, `tinyWinogrande`
+
+### Usage
+
+*tinyBenchmarks* can evaluate different benchmarks with a fraction of their examples.
+To obtain accurate results, this task applies post-processing using the *tinyBenchmarks*-package.
+You can install the package by running the following commands on the terminal (for more information see [here](https://github.com/felipemaiapolo/tinyBenchmarks/blob/main/README.md?plain=1)):
+
+``` :sh
+pip install git+https://github.com/felipemaiapolo/tinyBenchmarks
+```
+
+The value that is returned by the task corresponds to the '**IRT++**'-method from the [original paper](https://arxiv.org/abs/2402.14992).
+Evaluate specific tasks individually (e.g. `--tasks tinyHellaswag`) or all [open LLM leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) tasks by specifying `--tasks tinyBenchmarks`.
+
+### Advanced usage
+
+To obtain the estimated accuracies from all methods from the original paper, the *tinyBenchmarks*-package has to be applied manually.
+To do so, run the evaluation with the `--log_samples` and `--output_path` arguments. For example:
+
+```bash
+lm_eval --model hf  \
+        --model_args pretrained="mistralai/Mistral-7B-Instruct-v0.2" \
+        --tasks tinyHellaswag \
+        --batch_size 4 \
+        --output_path '<output_path>' \
+        --log_samples
+```
+
+Afterwards, run include the correct `file_path` and run the following script:
+
+```python
+import json
+import tinyBenchmarks as tb
+import numpy as np
+
+# Choose benchmark (e.g. hellaswag)
+benchmark = 'hellaswag' # possible benchmarks:
+                        # ['mmlu','truthfulqa', 'gsm8k',
+                        #  'winogrande', 'arc', 'hellaswag']
+
+# Get score vector from output-file (the metric [here `acc_norm`] depends on the benchmark)
+file_path = '<output_path>/<output-file.jsonl>'
+with open(file_path, 'r') as file:
+    outputs = json.load(file)
+
+# Ensuring correct order of outputs  
+outputs = sorted(outputs, key=lambda x: x['doc_id'])
+
+y = np.array([float(item['acc_norm']) for item in outputs])
+
+### Evaluation
+tb.evaluate(y, benchmark)
+```
+
+### Performance
+
+We report in the following tables the average estimation error in the test set (using data from the paper) and standard deviation across LLMs.
+
+#### Open LLM Leaderboard
+
+Estimating performance for each scenario separately
+|| IRT | p-IRT | gp-IRT |
+|--|--|--|--|
+| TruthfulQA | 0.013 (0.010) | 0.010 (0.009) | 0.011 (0.009) |
+| GSM8K | 0.022 (0.017) | 0.029 (0.022) | 0.020 (0.017) |
+| Winogrande | 0.022 (0.017) | 0.016 (0.014) | 0.015 (0.013) |
+| ARC | 0.022 (0.018) | 0.017 (0.014) | 0.017 (0.013) |
+| HellaSwag | 0.013 (0.016) | 0.015 (0.012) | 0.015 (0.012) |
+| MMLU | 0.024 (0.017) | 0.016 (0.015) | 0.016 (0.015) |
+
+Estimating performance for each scenario all at once
+|| IRT | p-IRT | gp-IRT |
+|--|--|--|--|
+| TruthfulQA  | 0.013 (0.010) | 0.016 (0.013) | 0.011 (0.009) |
+| GSM8K | 0.022 (0.017) | 0.022 (0.017) | 0.020 (0.015) |
+| Winogrande | 0.022 (0.017) | 0.011 (0.013) | 0.011 (0.011) |
+| ARC | 0.022 (0.018) | 0.012 (0.010) | 0.010 (0.009) |
+| HellaSwag | 0.013 (0.016) | 0.011 (0.020) | 0.011 (0.018) |
+| MMLU | 0.024 (0.018) | 0.017 (0.017) | 0.015 (0.015) |
+
+
+
+### Citation
+
+```
+@article{polo2024tinybenchmarks,
+      title={tinyBenchmarks: evaluating LLMs with fewer examples},
+      author={Maia Polo, Felipe and Weber, Lucas and Choshen, Leshem and Sun, Yuekai and Xu, Gongjun and Yurochkin, Mikhail},
+      journal={arXiv preprint arXiv:2402.14992},
+      year={2024}
+    }
+```
+
+Please also reference the respective original dataset that you are using!
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/tinyBenchmarks/agg_functions.py

+from typing import List
+
+import numpy as np
+
+
+try:
+    import tinyBenchmarks as tb
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "`tinyBenchmarks` is required for tinyBenchmarks task metric calculation, install via \
+`pip install git+https://github.com/felipemaiapolo/tinyBenchmarks`"
+    )
+
+
+def agg_pirt(items: List[float], benchmark: str) -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["pirt"]
+
+
+def agg_gpirt_arc(items: List[float], benchmark: str = "arc") -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["gpirt"]
+
+
+def agg_gpirt_gsm8k(items: List[float], benchmark: str = "gsm8k") -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["gpirt"]
+
+
+def agg_gpirt_hellaswag(items: List[float], benchmark: str = "hellaswag") -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["gpirt"]
+
+
+def agg_gpirt_mmlu(items: List[float], benchmark: str = "mmlu") -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["gpirt"]
+
+
+def agg_gpirt_truthfulqa(items: List[float], benchmark: str = "truthfulqa") -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["gpirt"]
+
+
+def agg_gpirt_winogrande(items: List[float], benchmark: str = "winogrande") -> float:
+    items = np.array(items)
+    predictions = tb.evaluate(items, benchmark)
+    return predictions[benchmark]["gpirt"]

lm_eval/tasks/tinyBenchmarks/tinyArc.yaml

+task: tinyArc
+dataset_path: tinyBenchmarks/tinyAI2_arc
+dataset_name: ARC-Challenge
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+num_fewshot: 25
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: acc_norm
+    aggregation: !function agg_functions.agg_gpirt_arc
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/tinyBenchmarks/tinyBenchmarks.yaml

+group: tinyBenchmarks
+task:
+  - task: tinyArc
+    num_fewshot: 25
+  - task: tinyGSM8k
+    num_fewshot: 5
+  - task: tinyMMLU
+    num_fewshot: 0
+  - task: tinyWinogrande
+    num_fewshot: 5
+  - task: tinyHellaswag
+    num_fewshot: 10
+  - task: tinyTruthfulQA
+    num_fewshot: 0
+metadata:
+  version: 0.0

lm_eval/tasks/tinyBenchmarks/tinyGSM8k.yaml

+task: tinyGSM8k
+dataset_path: tinyBenchmarks/tinyGSM8k
+dataset_name: main
+output_type: generate_until
+training_split: train
+fewshot_split: train
+test_split: test
+num_fewshot: 5
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{answer}}" #" {{answer.split('### ')[-1].rstrip()}}"
+metric_list:
+  - metric: exact_match
+    aggregation: !function agg_functions.agg_gpirt_gsm8k
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: false
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+      - "(?s).*#### "
+      - "\\.$"
+generation_kwargs:
+  until:
+    - "Question:"
+    - "</s>"
+    - "<|im_end|>"
+  do_sample: false
+  temperature: 0.0
+repeats: 1
+num_fewshot: 5
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+  - name: "flexible-extract"
+    filter:
+      - function: "regex"
+        group_select: -1
+        regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+      - function: "take_first"
+metadata:
+  version: 0.0