Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into multiprompt

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml

+"dataset_name": "Univ Accounting"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_template_yaml"
+"task": "arabicmmlu_univ_accounting"
+"task_alias": "Univ Accounting"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml

+"dataset_name": "Univ Computer Science"
+"tag": "arabicmmlu_stem_tasks"
+"include": "_default_template_yaml"
+"task": "arabicmmlu_univ_computer_science"
+"task_alias": "Univ Computer Science"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml

+"dataset_name": "Univ Economics"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_template_yaml"
+"task": "arabicmmlu_univ_economics"
+"task_alias": "Univ Economics"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml

+"dataset_name": "Univ Management"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_template_yaml"
+"task": "arabicmmlu_univ_management"
+"task_alias": "Univ Management"

lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml

+"dataset_name": "Univ Political Science"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_template_yaml"
+"task": "arabicmmlu_univ_political_science"
+"task_alias": "Univ Political Science"

lm_eval/tasks/arabicmmlu/utils.py

+PROMPT = "This is a {}. Select the correct answer!\n\nQuestion: {}\n{}\n\nAnswer:"
+
+level_en = {
+    "Primary": "primary school",
+    "Middle": "middle school",
+    "High": "high school",
+    "Univ": "university",
+    "Prof": "professional",
+}
+
+alpa = ["A.", "B.", "C.", "D.", "E."]
+
+
+def doc_to_text(doc):
+    """
+    Refactoring `prepare_data_en` to fit with the lm harness framework.
+    https://github.com/mbzuai-nlp/ArabicMMLU/blob/main/util_prompt.py
+    """
+
+    level = "" if not doc["Level"] else " for " + level_en[doc["Level"]]
+    country = "" if not doc["Country"] else " in " + doc["Country"]
+    main_meta_data = f"{doc['Subject']} question{level}{country}"
+
+    question = (
+        doc["Question"]
+        if doc["Context"] == ""
+        else f"{doc['Context']}\n\n{doc['Question']}"
+    )
+
+    options = []
+    for i, opt in enumerate(
+        ["Option 1", "Option 2", "Option 3", "Option 4", "Option 5"]
+    ):
+        if not doc[opt]:
+            break
+        options.append(f"{alpa[i]} {doc[opt]}")
+
+    doc_text = PROMPT.format(main_meta_data, question, "\n".join(options))
+
+    return doc_text
+
+
+def doc_to_choice(doc):
+    return [alpa[i][0] for i in range(5) if doc[f"Option {i+1}"]]

lm_eval/tasks/arc/README.md

@@ -29,10 +29,14 @@ Homepage: https://allenai.org/data/arc
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
+None.
+
+#### Tags
+
 * `ai2_arc`: Evaluates `arc_easy` and `arc_challenge`
 
 #### Tasks

lm_eval/tasks/arc/arc_easy.yaml

-group:
+tag:
   - ai2_arc
 task: arc_easy
 dataset_path: allenai/ai2_arc

lm_eval/tasks/arc_mt/README.md

+# arc mt
+
+arc mt is an implementation of tasks to support machine translated arc
+challenge evals, to improve eval support across a number of additional
+languages.
+
+The main page for the effort is
+[here](https://huggingface.co/datasets/LumiOpen/arc_challenge_mt) and we will
+include more data and analysis there.
+
+Initial datasets include a number of European languages, and we plan to expand
+more in the future.

lm_eval/tasks/arc_mt/arc_challenge_mt_da.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_da
+dataset_name: da

lm_eval/tasks/arc_mt/arc_challenge_mt_de.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_de
+dataset_name: de

lm_eval/tasks/arc_mt/arc_challenge_mt_el.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_el
+dataset_name: el

lm_eval/tasks/arc_mt/arc_challenge_mt_es.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_es
+dataset_name: es

lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml

+tag:
+  - arc_challenge_mt
+task: arc_challenge_mt_fi
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: fi
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arc_mt/arc_challenge_mt_hu.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_hu
+dataset_name: hu

lm_eval/tasks/arc_mt/arc_challenge_mt_is.yaml

+group:
+  - arc_challenge_mt
+task: arc_challenge_mt_is
+dataset_path: mideind/icelandic-arc-challenge
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/arc_mt/arc_challenge_mt_it.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_it
+dataset_name: it

lm_eval/tasks/arc_mt/arc_challenge_mt_nb.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_nb
+dataset_name: nb

lm_eval/tasks/arc_mt/arc_challenge_mt_pl.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_pl
+dataset_name: pl

lm_eval/tasks/arc_mt/arc_challenge_mt_pt.yaml

+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_pt
+dataset_name: pt