Add multilingual truthfulqa targets (#1499)

d272c19f · Zehan Li · GitHub · b177c82c · d272c19f
Unverified Commit d272c19f authored Mar 01, 2024 by Zehan Li Committed by GitHub Mar 01, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 0 deletions

lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py +1 -0

No files found.
--- a/lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py
+++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py
@@ -37,6 +37,7 @@ def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
            "query": QA_PROMPT + "\n\nQ: " + preprocess(doc["question"]) + "\nA:",
            "mc1_choices": doc["mc1_targets_choices"],
            "mc2_choices": doc["mc2_targets_choices"],
+            "mc2_targets": {"labels": doc["mc2_targets_labels"]},
            "gold": " ",
        }
        return out_doc