Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into superglue

lm_eval/tasks/crows_pairs/crows_pairs_french_race_color.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_race_color
+dataset_name: french
+process_docs: !function utils.filter_race_color

lm_eval/tasks/crows_pairs/crows_pairs_french_religion.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_religion
+dataset_name: french
+process_docs: !function utils.filter_religion

lm_eval/tasks/crows_pairs/crows_pairs_french_sexual_orientation.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_sexual_orientation
+dataset_name: french
+process_docs: !function utils.filter_orientation

lm_eval/tasks/crows_pairs/crows_pairs_french_socioeconomic.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_socioeconomic
+dataset_name: french
+process_docs: !function utils.filter_socio

lm_eval/tasks/crows_pairs/utils.py

+import datasets
+
+
+def process_results(doc, results):
+    lls, _ = zip(*results)
+
+    likelihood1, likelihood2 = lls
+
+    # Calculate the absolute difference in loglikelihoods
+    diff = abs(likelihood1 - likelihood2)
+
+    # if stereotypical sentence more likely (loglikelihood higher)
+    # then treat this as predicting stereotyped sentence
+    acc = 1.0 if likelihood1 > likelihood2 else 0.0
+
+    return {"likelihood_diff": diff, "acc": acc}
+
+
+def doc_to_choice(doc):
+    return [doc["sent_more"], doc["sent_less"]]
+
+
+def filter_dataset(dataset: datasets.Dataset, bias_type: str) -> datasets.Dataset:
+    return dataset.filter(lambda example: example["bias_type"].startswith(bias_type))
+
+
+def filter_race_color(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "race-color")
+
+
+def filter_socio(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "socioeconomic")
+
+
+def filter_gender(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "gender")
+
+
+def filter_age(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "age")
+
+
+def filter_religion(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "religion")
+
+
+def filter_disability(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "disability")
+
+
+def filter_orientation(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "sexual-orientation")
+
+
+def filter_nationality(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "nationality")
+
+
+def filter_appearance(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "physical-appearance")
+
+
+def filter_autre(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "autre")

lm_eval/tasks/logiqa/logiqa.yaml

+group:
+  - multiple_choice
+task: logiqa
+dataset_path: EleutherAI/logiqa
+dataset_name: logiqa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_choice: "{{options}}"
+doc_to_text: !function utils_logiqa.doc_to_text
+doc_to_target: !function utils_logiqa.doc_to_target
+doc_to_decontamination_query: "{{context}}"
+should_decontaminate: true
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/logiqa/utils_logiqa.py

+# Copied from Master
+def doc_to_text(doc) -> str:
+    """
+    Passage: <passage>
+    Question: <question>
+    Choices:
+    A. <choice1>
+    B. <choice2>
+    C. <choice3>
+    D. <choice4>
+    Answer:
+    """
+    choices = ["a", "b", "c", "d"]
+    prompt = "Passage: " + doc["context"] + "\n"
+    prompt += "Question: " + doc["question"] + "\nChoices:\n"
+    for choice, option in zip(choices, doc["options"]):
+        prompt += f"{choice.upper()}. {option}\n"
+    prompt += "Answer:"
+    return prompt
+
+
+def doc_to_target(doc) -> int:
+    choices = ["a", "b", "c", "d"]
+    return choices.index(doc["label"].strip())

lm_eval/tasks/logiqa2/README.md

+# LogiQA 2.0
+
+### Paper
+
+LogiQA 2.0 — An Improved Dataset for Logical Reasoning in Natural Language Understanding https://ieeexplore.ieee.org/document/10174688
+
+
+The dataset is an amendment and re-annotation of LogiQA in 2020, a large-scale logical reasoning reading comprehension dataset adapted from the Chinese Civil Service Examination. This new version has an increased data size, the texts are refined with manual translation by professionals, and improved by removing items with distinctive cultural features like Chinese idioms.
+
+Furthermore, a two-way natural language inference (NLI) task is introduced, resulting in 35k premise-hypothesis pairs with gold labels, making it the first large-scale NLI dataset for complex logical reasoning
+
+Homepage: https://github.com/csitfun/LogiQA2.0
+
+### Citation
+
+```bibtex
+@ARTICLE{10174688,
+  author={Liu, Hanmeng and Liu, Jian and Cui, Leyang and Teng, Zhiyang and Duan, Nan and Zhou, Ming and Zhang, Yue},
+  journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing},
+  title={LogiQA 2.0 — An Improved Dataset for Logical Reasoning in Natural Language Understanding},
+  year={2023},
+  volume={},
+  number={},
+  pages={1-16},
+  doi={10.1109/TASLP.2023.3293046}}
+```
+
+### Subtasks
+
+`logiqa2_zh`: The original dataset in Chinese.
+
+`logiqa2_NLI`: The NLI version of the dataset converted from the MRC version.
+
+`logieval`: Prompt based; https://github.com/csitfun/LogiEval
+
+The subtasks have not been verified yet.
+
+### Checklist
+
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation?
+    * [x] The original paper does not. There is another implementation of this task, but it designed for instruction tuned models: https://github.com/csitfun/LogiEval
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/logiqa2/logieval.yaml

+group:
+  - greedy_until
+task: logieval
+dataset_path: baber/logiqa2
+dataset_name: logieval
+output_type: greedy_until
+training_split: train
+test_split: test
+# Instructions + {content}
+doc_to_text: "Instructions: You will be presented with a passage and a question about that passage. There are four options to be chosen from, you need to choose the only correct option to answer that question. If the first option is right, you generate the answer 'A', if the second option is right, you generate the answer 'B', if the third option is right, you generate the answer 'C', if the fourth option is right, you generate the answer 'D'. Read the question and options thoroughly and select the correct answer from the four answer labels. Read the passage thoroughly to ensure you know what the passage entails.\n{{content}}"
+doc_to_target: "{{ideal}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+generation_kwargs:
+  do_sample: false
+num_fewshot: 1
+filter_list:
+  - name: "get-answer"
+    filter:
+      - function: "regex"
+        # starts with A-D excluding leading spaces
+        # original implementation uses a.startswith(b)
+        # https://github.com/openai/evals/blob/305b237cdb3884c7ddb6a5d12cb184a83551fcba/evals/api.py#L84
+        regex_pattern: "^\\s*([A-D])"
+      - function: "take_first"

lm_eval/tasks/logiqa2/logiqa2.yaml

+group:
+  - multiple_choice
+task: logiqa2
+dataset_path: baber/logiqa2
+dataset_name: logiqa2
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_choice: "{{options}}"
+doc_to_text: !function utils_logiqa2.doc_to_text
+doc_to_target: "{{answer}}"
+doc_to_decontamination_query: "{{context}}"
+should_decontaminate: false
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/logiqa2/utils_logiqa2.py

+# Copied from Master
+def doc_to_text(doc) -> str:
+    """
+    Passage: <passage>
+    Question: <question>
+    A. <choice1>
+    B. <choice2>
+    C. <choice3>
+    D. <choice4>
+    Answer:
+    """
+    choices = ["a", "b", "c", "d"]
+    prompt = "Passage: " + doc["text"] + "\n"
+    prompt += "Question: " + doc["question"] + "\n"
+    for choice, option in zip(choices, doc["options"]):
+        prompt += f"{choice.upper()}. {option}\n"
+    prompt += "Answer:"
+    return prompt
+
+
+# # https://github.com/csitfun/LogiQA2.0/blob/main/logiqa2nli/nli-prompt.py
+# def doc_to_textNLI(doc):
+#     maj_premise = ' '.join(list(doc['major_premise']))
+#     min_premise = ' '.join(list(doc['minor_premise']))
+#     hypo = doc['conclusion']
+#     prompt_input = "Given the fact: " + maj_premise + ' ' + min_premise + " Does it follow that: " + hypo + " Yes or no?"
+#     return prompt_input

lm_eval/tasks/xcopa/README.md

+## XCOPA: A Multilingual Dataset for Causal Commonsense Reasoning
+https://ducdauge.github.io/files/xcopa.pdf
+
+The Cross-lingual Choice of Plausible Alternatives dataset is a benchmark to evaluate the ability of machine learning models to transfer commonsense reasoning across languages.
+The dataset is the translation and reannotation of the English COPA (Roemmele et al. 2011) and covers 11 languages from 11 families and several areas around the globe.
+The dataset is challenging as it requires both the command of world knowledge and the ability to generalise to new languages.
+All the details about the creation of XCOPA and the implementation of the baselines are available in the paper.
+
+Homepage: https://github.com/cambridgeltl/xcopa
+
+```
+@inproceedings{ponti2020xcopa,
+  title={{XCOPA: A} Multilingual Dataset for Causal Commonsense Reasoning},
+  author={Edoardo M. Ponti, Goran Glava\v{s}, Olga Majewska, Qianchu Liu, Ivan Vuli\'{c} and Anna Korhonen},
+  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
+  year={2020},
+  url={https://ducdauge.github.io/files/xcopa.pdf}
+}
+```

lm_eval/tasks/xcopa/default_et.yaml

+group: xcopa
+task: xcopa_et
+dataset_path: xcopa
+dataset_name: et
+output_type: multiple_choice
+validation_split: validation
+test_split: test
+doc_to_text: !function utils.doc_to_text_et
+doc_to_target: label
+doc_to_choice: !function utils.doc_to_choice
+metric_list:
+  - metric: acc

lm_eval/tasks/xcopa/default_ht.yaml

+include: default_et.yaml
+task: xcopa_ht
+dataset_name: ht
+doc_to_text: !function utils.doc_to_text_ht

lm_eval/tasks/xcopa/default_id.yaml

+include: default_et.yaml
+task: xcopa_id
+dataset_name: id
+doc_to_text: !function utils.doc_to_text_id

lm_eval/tasks/xcopa/default_it.yaml

+include: default_et.yaml
+task: xcopa_it
+dataset_name: it
+doc_to_text: !function utils.doc_to_text_it

lm_eval/tasks/xcopa/default_qu.yaml

+include: default_et.yaml
+task: xcopa_qu
+dataset_name: qu
+doc_to_text: !function utils.doc_to_text_qu

lm_eval/tasks/xcopa/default_sw.yaml

+include: default_et.yaml
+task: xcopa_sw
+dataset_name: sw
+doc_to_text: !function utils.doc_to_text_sw

lm_eval/tasks/xcopa/default_ta.yaml

+include: default_et.yaml
+task: xcopa_ta
+dataset_name: ta
+doc_to_text: !function utils.doc_to_text_ta

lm_eval/tasks/xcopa/default_th.yaml

+include: default_et.yaml
+task: xcopa_th
+dataset_name: th
+doc_to_text: !function utils.doc_to_text_th