Merge branch 'main' into fix-task-table

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_ro.yaml

+include: _hellaswag_yaml
+task: hellaswag_ro
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: ro
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_ru.yaml

+include: _hellaswag_yaml
+task: hellaswag_ru
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: ru
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sk.yaml

+include: _hellaswag_yaml
+task: hellaswag_sk
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: sk
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sr.yaml

+include: _hellaswag_yaml
+task: hellaswag_sr
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: sr
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sv.yaml

+include: _hellaswag_yaml
+task: hellaswag_sv
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: sv
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_ta.yaml

+include: _hellaswag_yaml
+task: hellaswag_ta
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: ta
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_te.yaml

+include: _hellaswag_yaml
+task: hellaswag_te
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: te
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_uk.yaml

+include: _hellaswag_yaml
+task: hellaswag_uk
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: uk
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_vi.yaml

+include: _hellaswag_yaml
+task: hellaswag_vi
+dataset_path: alexandrainst/m_hellaswag
+dataset_name: vi
+training_split: null
+validation_split: val

lm_eval/tasks/okapi/hellaswag_multilingual/utils.py

+import datasets
+import re
+
+
+def preprocess(text):
+    text = text.strip()
+    # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
+    text = text.replace(" [title]", ". ")
+    text = re.sub("\\[.*?\\]", "", text)
+    text = text.replace("  ", " ")
+    return text
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _process_doc(doc):
+        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
+        out_doc = {
+            "query": preprocess(doc["activity_label"] + ": " + ctx),
+            "choices": [preprocess(ending) for ending in doc["endings"]],
+            "gold": int(doc["label"]),
+        }
+        return out_doc
+
+    return dataset.map(_process_doc)

lm_eval/tasks/polemo2/polemo2_in.yaml

@@ -2,7 +2,7 @@ group:
   - polemo2
 task: polemo2_in
 dataset_path: allegro/klej-polemo2-in
-dataset_name: klej-polemo2-in
+dataset_name: null
 output_type: generate_until
 training_split: train
 validation_split: validation
@@ -41,5 +41,6 @@ metric_list:
   - metric: accuracy
     aggregation: mean
     higher_is_better: true
+    hf_evaluate: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/qasper/freeform.yaml

@@ -15,4 +15,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/race/README.md

@@ -17,7 +17,25 @@ Homepage: https://www.cs.cmu.edu/~glai1/data/race/
 ### Citation
 
 ```
-BibTeX-formatted citation goes here
+@inproceedings{lai-etal-2017-race,
+    title = "{RACE}: Large-scale {R}e{A}ding Comprehension Dataset From Examinations",
+    author = "Lai, Guokun  and
+      Xie, Qizhe  and
+      Liu, Hanxiao  and
+      Yang, Yiming  and
+      Hovy, Eduard",
+    editor = "Palmer, Martha  and
+      Hwa, Rebecca  and
+      Riedel, Sebastian",
+    booktitle = "Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing",
+    month = sep,
+    year = "2017",
+    address = "Copenhagen, Denmark",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/D17-1082",
+    doi = "10.18653/v1/D17-1082",
+    pages = "785--794"
+}
 ```
 
 ### Groups and Tasks

lm_eval/tasks/scrolls/task.py

@@ -108,7 +108,7 @@ def _num_cpu_cores():
 
 
 class _SCROLLSTask(Task):
-    VERSION = 1
+    VERSION = 2
     DATASET_PATH = "tau/scrolls"
     DATASET_NAME = None
     PRUNE_TOKENIZERS = None

lm_eval/tasks/squadv2/task.py

@@ -49,7 +49,7 @@ def _squad_agg(key, items):
 
 @register_task("squadv2")
 class SQuAD2(Task):
-    VERSION = 2
+    VERSION = 3
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 

lm_eval/tasks/translation/wmt_common_yaml

@@ -14,4 +14,4 @@ generation_kwargs:
   temperature: 0.0
 repeats: 1
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/triviaqa/default.yaml

@@ -28,4 +28,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -76,4 +76,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -17,4 +17,4 @@ metric_list:
     ignore_case: false
     ignore_punctuation: false
 metadata:
-  version: 1.0
+  version: 2.0