Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into multiprompt

lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml

 "dataset_name": "professional_psychology"
 "description": "The following are multiple choice questions (with answers) about professional\
   \ psychology.\n\n"
-"group": "mmlu_social_sciences_generative"
-"group_alias": "social_sciences"
+"tag": "mmlu_social_sciences_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_psychology_generative"
 "task_alias": "professional_psychology"

lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml

 "dataset_name": "public_relations"
 "description": "The following are multiple choice questions (with answers) about public\
   \ relations.\n\n"
-"group": "mmlu_social_sciences_generative"
-"group_alias": "social_sciences"
+"tag": "mmlu_social_sciences_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_public_relations_generative"
 "task_alias": "public_relations"

lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml

 "dataset_name": "security_studies"
 "description": "The following are multiple choice questions (with answers) about security\
   \ studies.\n\n"
-"group": "mmlu_social_sciences_generative"
-"group_alias": "social_sciences"
+"tag": "mmlu_social_sciences_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_security_studies_generative"
 "task_alias": "security_studies"

lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml

 "dataset_name": "sociology"
 "description": "The following are multiple choice questions (with answers) about sociology.\n\
   \n"
-"group": "mmlu_social_sciences_generative"
-"group_alias": "social_sciences"
+"tag": "mmlu_social_sciences_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_sociology_generative"
 "task_alias": "sociology"

lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml

 "dataset_name": "us_foreign_policy"
 "description": "The following are multiple choice questions (with answers) about us\
   \ foreign policy.\n\n"
-"group": "mmlu_social_sciences_generative"
-"group_alias": "social_sciences"
+"tag": "mmlu_social_sciences_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_us_foreign_policy_generative"
 "task_alias": "us_foreign_policy"

lm_eval/tasks/mmlu/generative/mmlu_virology.yaml

 "dataset_name": "virology"
 "description": "The following are multiple choice questions (with answers) about virology.\n\
   \n"
-"group": "mmlu_other_generative"
-"group_alias": "other"
+"tag": "mmlu_other_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_virology_generative"
 "task_alias": "virology"

lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml

 "dataset_name": "world_religions"
 "description": "The following are multiple choice questions (with answers) about world\
   \ religions.\n\n"
-"group": "mmlu_humanities_generative"
-"group_alias": "humanities"
+"tag": "mmlu_humanities_generative"
 "include": "_default_template_yaml"
 "task": "mmlu_world_religions_generative"
 "task_alias": "world_religions"

lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml

-group: advanced_ai_risk
+tag: advanced_ai_risk
 dataset_path: EleutherAI/advanced_ai_risk
 output_type: multiple_choice
 validation_split: validation

lm_eval/tasks/model_written_evals/persona/_template_yaml

-group: persona
+tag: persona
 dataset_path: EleutherAI/persona
 output_type: multiple_choice
 validation_split: validation

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

-group: sycophancy
+tag: sycophancy
 task: sycophancy_on_nlp_survey
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_nlp_survey

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

-group: sycophancy
+tag: sycophancy
 task: sycophancy_on_philpapers2020
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_philpapers2020

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

-group: sycophancy
+tag: sycophancy
 task: sycophancy_on_political_typology_quiz
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_political_typology_quiz

lm_eval/tasks/model_written_evals/winogenerated/_template_yaml → lm_eval/tasks/model_written_evals/winogenerated/winogenerated.yaml

-group: winogenerated
+tag: winogenerated
 dataset_path: EleutherAI/winogenerated
 output_type: multiple_choice
 validation_split: validation

lm_eval/tasks/noticia/README.md

+# NoticIA
+
+### Paper
+
+Title: `NoticIA: A Clickbait Article Summarization Dataset in Spanish`
+
+Abstract: https://arxiv.org/abs/2404.07611
+
+We present NoticIA, a dataset consisting of 850 Spanish news articles featuring prominent clickbait headlines, each paired with high-quality, single-sentence generative summarizations written by humans. This task demands advanced text understanding and summarization abilities, challenging the models' capacity to infer and connect diverse pieces of information to meet the user's informational needs generated by the clickbait headline. We evaluate the Spanish text comprehension capabilities of a wide range of state-of-the-art large language models. Additionally, we use the dataset to train ClickbaitFighter, a task-specific model that achieves near-human performance in this task.
+
+Homepage: https://github.com/ikergarcia1996/NoticIA
+
+### Citation
+
+```
+@article{noticia2024,
+      title={NoticIA: A Clickbait Article Summarization Dataset in Spanish},
+      author={Iker García-Ferrero and Begoña Altuna},
+      year={2024},
+      journal = {Procesamiento del Lenguaje Natural},
+      volume = {73},
+      number = {0},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* Not part of a group yet.
+
+#### Tasks
+
+* `noticia`
+
+#### Metrics
+
+Following the original implementation, this task will compute the 'Rouge1 score' and 'Average Summary Length.'
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/noticia/noticia.yaml

+task: noticia
+dataset_path: Iker/NoticIA
+dataset_name: null
+output_type: generate_until
+generation_kwargs:
+  until:
+    - "\n\n"
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+training_split: null
+validation_split: null
+test_split: test
+fewshot_split: null
+doc_to_text: "Ahora eres una Inteligencia Artificial experta en desmontar titulares sensacionalistas o clickbait. Tu tarea consiste en analizar noticias con titulares sensacionalistas y generar un resumen de una sola frase que revele la verdad detrás del titular.\nEste es el titular de la noticia: {{web_headline}}\nEl titular plantea una pregunta o proporciona información incompleta. Debes buscar en el cuerpo de la noticia una frase que responda lo que se sugiere en el título. Responde siempre que puedas parafraseando el texto original. Usa siempre las mínimas palabras posibles. Recuerda responder siempre en Español.\nEste es el cuerpo de la noticia:\n{{web_text}}"
+doc_to_target: summary
+target_delimiter: " "
+num_fewshot: 0
+should_decontaminate: false
+doc_to_decontamination_query: sentence
+metric_list:
+  - metric: !function utils.rouge1
+    higher_is_better: true
+    aggregation: !function utils.rouge1_agg
+  - metric: !function utils.average_len
+    higher_is_better: false
+    aggregation: !function utils.average_len_agg
+metadata:
+  version: 1.0

lm_eval/tasks/noticia/utils.py

+import string
+
+import evaluate
+
+
+def clean_text(text: str) -> str:
+    # Remove punctuation
+    text = text.translate(str.maketrans("", "", string.punctuation))
+
+    # Remove newlines and multiple spaces
+    text = text.replace("\n", " ").strip()
+    text = " ".join(text.split()).strip()
+
+    # lowercase
+    text = text.lower()
+
+    return text
+
+
+def rouge1(items):
+    """
+    # passthrough for efficiency
+    """
+    return items
+
+
+def average_len(items):
+    """
+    # passthrough for efficiency
+    """
+    return items
+
+
+def rouge1_agg(items):
+    """
+    Higher is better
+    """
+
+    refs = list(zip(*items))[0]
+    refs = [[clean_text(ref)] for ref in refs]
+    # print("refs", refs)
+    preds = [clean_text(x) for x in list(zip(*items))[1]]
+    # print("preds", preds)
+    rouge_scorer = evaluate.load("rouge")
+    return rouge_scorer.compute(predictions=preds, references=refs)["rouge1"]
+
+
+def average_len_agg(items):
+    """
+    Higher is better
+    """
+
+    preds = [clean_text(x) for x in list(zip(*items))[1]]
+
+    return sum(len(x.split()) for x in preds) / len(preds)

lm_eval/tasks/nq_open/README.md

@@ -14,24 +14,6 @@ Derived from the Natural Questions dataset, introduced in https://storage.google
 ### Citation
 
 ```
-@inproceedings{lee-etal-2019-latent,
-    title = "Latent Retrieval for Weakly Supervised Open Domain Question Answering",
-    author = "Lee, Kenton  and
-      Chang, Ming-Wei  and
-      Toutanova, Kristina",
-    editor = "Korhonen, Anna  and
-      Traum, David  and
-      M{\`a}rquez, Llu{\'\i}s",
-    booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics",
-    month = jul,
-    year = "2019",
-    address = "Florence, Italy",
-    publisher = "Association for Computational Linguistics",
-    url = "https://aclanthology.org/P19-1612",
-    doi = "10.18653/v1/P19-1612",
-    pages = "6086--6096",
-    abstract = "Recent work on open domain question answering (QA) assumes strong supervision of the supporting evidence and/or assumes a blackbox information retrieval (IR) system to retrieve evidence candidates. We argue that both are suboptimal, since gold evidence is not always available, and QA is fundamentally different from IR. We show for the first time that it is possible to jointly learn the retriever and reader from question-answer string pairs and without any IR system. In this setting, evidence retrieval from all of Wikipedia is treated as a latent variable. Since this is impractical to learn from scratch, we pre-train the retriever with an Inverse Cloze Task. We evaluate on open versions of five QA datasets. On datasets where the questioner already knows the answer, a traditional IR system such as BM25 is sufficient. On datasets where a user is genuinely seeking an answer, we show that learned retrieval is crucial, outperforming BM25 by up to 19 points in exact match.",
-}
 @article{47761,
 title	= {Natural Questions: a Benchmark for Question Answering Research},
 author	= {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},

lm_eval/tasks/okapi/arc_multilingual/_arc_yaml

-group:
+tag:
   - arc_multilingual
 dataset_path: null
 dataset_name: null

lm_eval/tasks/okapi/hellaswag_multilingual/_hellaswag_yaml

-group:
+tag:
   - hellaswag_multilingual
 dataset_path: null
 dataset_name: null

lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml

-group:
+tag:
   - m_mmlu
 dataset_path: alexandrainst/m_mmlu
 test_split: test