Merge branch 'big-refactor' of...

Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into flan-benchmark

lm_eval/tasks/bigbench/multiple_choice/undo_permutation.yaml

+# Generated by utils.py
+dataset_name: undo_permutation_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_undo_permutation_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/unit_conversion.yaml

+# Generated by utils.py
+dataset_name: unit_conversion_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_unit_conversion_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/unit_interpretation.yaml

+# Generated by utils.py
+dataset_name: unit_interpretation_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_unit_interpretation_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/unnatural_in_context_learning.yaml

+# Generated by utils.py
+dataset_name: unnatural_in_context_learning_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_unnatural_in_context_learning_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/vitaminc_fact_verification.yaml

+# Generated by utils.py
+dataset_name: vitaminc_fact_verification_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_vitaminc_fact_verification_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/what_is_the_tao.yaml

+# Generated by utils.py
+dataset_name: what_is_the_tao_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_what_is_the_tao_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/which_wiki_edit.yaml

+# Generated by utils.py
+dataset_name: which_wiki_edit_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_which_wiki_edit_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/winowhy.yaml

+# Generated by utils.py
+dataset_name: winowhy_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_winowhy_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/word_sorting.yaml

+# Generated by utils.py
+dataset_name: word_sorting_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_word_sorting_multiple_choice

lm_eval/tasks/bigbench/multiple_choice/word_unscrambling.yaml

+# Generated by utils.py
+dataset_name: word_unscrambling_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_word_unscrambling_multiple_choice

lm_eval/tasks/bigbench/multiple_choice_template_yaml

+group: bigbench
+dataset_path: bigbench # will switch to `hails/bigbench` when all tasks are pushed
+dataset_kwargs:
+  # num_shots: 0 # TODO: num of shots for `bigbench` HF dataset should be controlled through this, not through the typical methods
+  # subtask_name: null
+output_type: multiple_choice
+test_split: default
+doc_to_text: inputs
+doc_to_target: "{{multiple_choice_targets.index(targets[0])}}"
+doc_to_choice: "{{multiple_choice_targets}}"
+metric_list:
+  - metric: acc
+  # TODO: brier score and other metrics

lm_eval/tasks/bigbench/push_bigbench_dataset.py

+"""
+A utility script that pushes all Bigbench subtasks from their form in the `bigbench` HF dataset
+into `{org name}/bigbench`.
+
+Prior to running, log into HF Hub for the target HF hub org via `huggingface-cli login`.
+
+Requires the installation of
+`pip install "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz"`
+and is included so that the bigbench dependency can be avoided.
+"""
+from tqdm import tqdm
+import datasets
+
+import bigbench.api.util as bb_utils
+
+
+all_task_names = bb_utils.get_all_json_task_names()
+
+num_shots = [0]
+
+for shots in num_shots:
+    for task_name in tqdm(all_task_names):
+        try:
+            print(f"Loading '{task_name}' with num_shots={shots}...")
+            task_ds = datasets.load_dataset("bigbench", name=task_name, num_shots=shots)
+
+            print(f"Pushing '{task_name}' with num_shots={shots}...")
+            task_ds.push_to_hub("hails/bigbench", task_name + "_zero_shot")
+
+            del task_ds
+        except Exception as e:
+            raise e

lm_eval/tasks/polemo2/README.md

+# PolEmo 2.0
+
+### Paper
+
+Title: `Multi-Level Sentiment Analysis of PolEmo 2.0: Extended Corpus of Multi-Domain Consumer Reviews`
+
+Abstract: https://aclanthology.org/K19-1092/
+
+The PolEmo 2.0 is a dataset of online consumer reviews in Polish from four domains: medicine, hotels, products, and university. It is human-annotated on a level of full reviews and individual sentences. It comprises over 8000 reviews, about 85% from the medicine and hotel domains.
+The goal is to predict the sentiment of a review. There are two separate test sets, to allow for in-domain (medicine and hotels) as well as out-of-domain (products and university) validation.
+
+Homepage: https://clarin-pl.eu/dspace/handle/11321/710
+
+
+### Citation
+
+```
+@inproceedings{kocon-etal-2019-multi,
+    title = "Multi-Level Sentiment Analysis of {P}ol{E}mo 2.0: Extended Corpus of Multi-Domain Consumer Reviews",
+    author = "Koco{\'n}, Jan  and
+      Mi{\l}kowski, Piotr  and
+      Za{\'s}ko-Zieli{\'n}ska, Monika",
+    booktitle = "Proceedings of the 23rd Conference on Computational Natural Language Learning (CoNLL)",
+    month = nov,
+    year = "2019",
+    address = "Hong Kong, China",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/K19-1092",
+    doi = "10.18653/v1/K19-1092",
+    pages = "980--991",
+    abstract = "In this article we present an extended version of PolEmo {--} a corpus of consumer reviews from 4 domains: medicine, hotels, products and school. Current version (PolEmo 2.0) contains 8,216 reviews having 57,466 sentences. Each text and sentence was manually annotated with sentiment in 2+1 scheme, which gives a total of 197,046 annotations. We obtained a high value of Positive Specific Agreement, which is 0.91 for texts and 0.88 for sentences. PolEmo 2.0 is publicly available under a Creative Commons copyright license. We explored recent deep learning approaches for the recognition of sentiment, such as Bi-directional Long Short-Term Memory (BiLSTM) and Bidirectional Encoder Representations from Transformers (BERT).",
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `polemo2`: Evaluates `polemo2_in` and `polemo2_out`
+
+#### Tasks
+
+* `polemo2_in`: evaluates sentiment predictions of in-domain (medicine and hotels) reviews
+* `polemo2_out`: evaluates sentiment predictions of out-of-domain (products and university) reviews
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/polemo2/polemo2_in.yaml

+group:
+  - polemo2
+task: polemo2_in
+dataset_path: allegro/klej-polemo2-in
+dataset_name: klej-polemo2-in
+output_type: greedy_until
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Opinia: \"{{sentence}}\"\nOkreśl sentyment podanej opinii. Możliwe odpowiedzi:\nA - Neutralny\nB - Negatywny\nC - Pozytywny\nD - Niejednoznaczny\nPrawidłowa odpowiedź:"
+doc_to_target: "{{['__label__meta_zero', '__label__meta_minus_m', '__label__meta_plus_m', '__label__meta_amb'].index(target)}}"
+should_decontaminate: true
+doc_to_decontamination_query: "{{sentence}}"
+generation_kwargs:
+  until:
+    - "."
+    - ","
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 50
+filter_list:
+  - name: "score-first"
+    filter:
+      - function: "regex"
+        regex_pattern: "(\\b[ABCD]\\b)"
+      - function: "take_first"
+      - function: "map"
+        mapping_dict:
+          A: 0
+          B: 1
+          C: 2
+          D: 3
+        default_value: -1
+      - function: "take_first"
+metric_list:
+  - metric: f1
+    aggregation: mean
+    higher_is_better: true
+    hf_evaluate: true
+    average: micro
+  - metric: accuracy
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/polemo2/polemo2_out.yaml

+include: polemo2_in.yaml
+task: polemo2_out
+dataset_path: allegro/klej-polemo2-out
+dataset_name: klej-polemo2-out

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

 def doc_to_text(doc) -> str:
-    ctxs = "\n".join(doc["context"]["contexts"])
+    ctxs = "\n".join(doc["CONTEXTS"])
     return "Abstract: {}\nQuestion: {}\nAnswer:".format(
-        ctxs, doc["question"], doc["final_decision"]
+        ctxs, doc["QUESTION"], doc["final_decision"]
     )
 
 

lm_eval/tasks/pubmedqa/pubmedqa.yaml

-task: pubmed_qa
-dataset_path: pubmed_qa
-dataset_name: pqa_labeled
+task: pubmedqa
+dataset_path: bigbio/pubmed_qa
+dataset_name: pubmed_qa_labeled_fold0_source
 output_type: multiple_choice
-training_split: null
-validation_split: null
-test_split: train
+training_split: train
+validation_split: validation
+test_split: test
 doc_to_text: !function preprocess_pubmedqa.doc_to_text
 doc_to_target: final_decision
 doc_to_choice: ["yes", "no", "maybe"]

pyproject.toml

@@ -47,8 +47,8 @@ lm_eval = ["**/*.yaml", "tasks/**/*"]
 examples = ["**/*.yaml"]
 
 [project.scripts]
-lm-eval = "main:main"
-lm_eval = "main:main"
+lm-eval = "lm_eval.__main__:cli_evaluate"
+lm_eval = "lm_eval.__main__:cli_evaluate"
 
 [project.urls]
 Homepage = "https://github.com/EleutherAI/lm-evaluation-harness"