Merge branch 'main' into add-chat-templating

lm_eval/tasks/code_x_glue/code-text/java.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/javascript.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/php.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/python.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/code_x_glue/code-text/ruby.yaml

@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/coqa/default.yaml

@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/drop/default.yaml

@@ -21,4 +21,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/fld/fld_default.yaml

@@ -18,4 +18,4 @@ filter_list:
       - function: remove_whitespace
       - function: take_first
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml

@@ -31,4 +31,4 @@ filter_list:
       - function: "majority_vote"
       - function: "take_first"
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/gsm8k/gsm8k-cot.yaml

@@ -41,4 +41,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)."
       - function: "take_first"
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/gsm8k/gsm8k.yaml

@@ -34,4 +34,4 @@ filter_list:
         regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/ifeval/ifeval.yaml

@@ -26,4 +26,4 @@ metric_list:
     aggregation: !function utils.agg_inst_level_acc
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/medmcqa/medmcqa.yaml

+task: medmcqa
+dataset_path: medmcqa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: validation
+doc_to_text: !function utils_medmcqa.doc_to_text
+doc_to_target: cop
+doc_to_choice: [ 'A','B','C','D' ]
+should_decontaminate: true
+doc_to_decontamination_query: "{{question}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/medmcqa/utils_medmcqa.py

+# Copied from Master
+def doc_to_text(doc) -> str:
+    """
+    Question: <question>
+    Choices:
+    A. <choice1>
+    B. <choice2>
+    C. <choice3>
+    D. <choice4>
+    Answer:
+    """
+    choices = [doc["opa"], doc["opb"], doc["opc"], doc["opd"]]
+    option_choices = {'A': choices[0], 'B': choices[1], 'C': choices[2], 'D': choices[3]}
+
+    prompt = "Question: " + doc["question"] + "\nChoices:\n"
+    for choice, option in option_choices.items():
+        prompt += f"{choice.upper()}. {option}\n"
+    prompt += "Answer:"
+    return prompt

lm_eval/tasks/medqa/medqa.yaml

+task: medqa_4options
+dataset_path: GBaker/MedQA-USMLE-4-options-hf
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: !function preprocess_medqa.doc_to_text
+doc_to_target: !function preprocess_medqa.doc_to_target
+doc_to_choice: [ 'A', 'B', 'C', 'D' ]
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/medqa/preprocess_medqa.py

+def doc_to_text(doc) -> str:
+    option_choices = {'A': doc["ending0"], 'B': doc["ending1"], 'C': doc["ending2"], 'D': doc["ending3"]}
+    answers = "".join((f"{k}. {v}\n") for k, v in option_choices.items())
+    return f"Question: {doc['sent1']}\n{answers}Answer:"
+
+
+def doc_to_target(doc) -> int:
+    return doc["label"]

lm_eval/tasks/mgsm/direct/direct_yaml

@@ -26,4 +26,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 0.0
+  version: 1.0

lm_eval/tasks/mgsm/native_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -21,4 +21,4 @@ metric_list:
     higher_is_better: true
 num_fewshot: 0
 metadata:
-  version: 0.0
+  version: 1.0