Merge branch 'smolrefact' into tasklist

# Conflicts:
#	lm_eval/__main__.py
#	lm_eval/api/group.py
#	lm_eval/api/task.py
#	lm_eval/evaluator_utils.py
#	lm_eval/tasks/__init__.py
#	lm_eval/utils.py
#	pyproject.toml

lm_eval/tasks/longbench/narrativeqa.yaml

@@ -5,17 +5,17 @@ task: longbench_narrativeqa
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: narrativeqa
-doc_to_text: 'You are given a story, which can be either a novel or a movie script, and a question. Answer the question asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {{context}}\n\nNow, answer the question based on the story asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
+doc_to_text: "You are given a story, which can be either a novel or a movie script, and a question. Answer the question asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {{context}}\n\nNow, answer the question based on the story asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/passage_count.yaml

@@ -5,17 +5,17 @@ task: longbench_passage_count
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_count
-doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '
+doc_to_text: "There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: "
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_count_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "count_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/passage_count_e.yaml

@@ -5,17 +5,17 @@ task: longbench_passage_count_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_count_e
-doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '
+doc_to_text: "There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: "
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_count_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "count_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/passage_retrieval_en.yaml

@@ -5,17 +5,17 @@ task: longbench_passage_retrieval_en
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_retrieval_en
-doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: '
+doc_to_text: "Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: "
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_retrieval_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "retrieval_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/passage_retrieval_en_e.yaml

@@ -5,17 +5,17 @@ task: longbench_passage_retrieval_en_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_retrieval_en_e
-doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: '
+doc_to_text: "Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: "
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_retrieval_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "retrieval_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/passage_retrieval_zh.yaml

@@ -5,17 +5,17 @@ task: longbench_passage_retrieval_zh
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_retrieval_zh
-doc_to_text: '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{{context}}\n\n下面是一个摘要\n\n{{input}}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：'
+doc_to_text: "以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{{context}}\n\n下面是一个摘要\n\n{{input}}\n\n请输入摘要所属段落的编号。答案格式必须是\"段落1\"，\"段落2\"等格式\n\n答案是："
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_retrieval_zh_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "retrieval_zh_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/qasper.yaml

@@ -5,17 +5,17 @@ task: longbench_qasper
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: qasper
-doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
+doc_to_text: "You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write \"unanswerable\". If the question is a yes/no question, answer \"yes\", \"no\", or \"unanswerable\". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write \"unanswerable\". If the question is a yes/no question, answer \"yes\", \"no\", or \"unanswerable\". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/qasper_e.yaml

@@ -5,17 +5,17 @@ task: longbench_qasper_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: qasper_e
-doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
+doc_to_text: "You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write \"unanswerable\". If the question is a yes/no question, answer \"yes\", \"no\", or \"unanswerable\". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write \"unanswerable\". If the question is a yes/no question, answer \"yes\", \"no\", or \"unanswerable\". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/qmsum.yaml

@@ -5,17 +5,17 @@ task: longbench_qmsum
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: qmsum
-doc_to_text: 'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{{context}}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {{input}}\nAnswer:'
+doc_to_text: "You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{{context}}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {{input}}\nAnswer:"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_rouge_score
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "rouge_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/repobench-p.yaml

@@ -5,17 +5,17 @@ task: longbench_repobench-p
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: repobench-p
-doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n'
+doc_to_text: "Please complete the code given below. \n{{context}}{{input}}Next line of code:\n"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_code_sim_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "code_sim_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/repobench-p_e.yaml

@@ -5,17 +5,17 @@ task: longbench_repobench-p_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: repobench-p_e
-doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n'
+doc_to_text: "Please complete the code given below. \n{{context}}{{input}}Next line of code:\n"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_code_sim_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "code_sim_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/samsum.yaml

@@ -5,17 +5,17 @@ task: longbench_samsum
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: samsum
-doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}'
+doc_to_text: "Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_rouge_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: ["\n"]
 metric_list:
   - metric: "rouge_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/samsum_e.yaml

@@ -5,17 +5,17 @@ task: longbench_samsum_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: samsum_e
-doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}'
+doc_to_text: "Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_rouge_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: ["\n"]
 metric_list:
   - metric: "rouge_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/trec.yaml

@@ -5,17 +5,17 @@ task: longbench_trec
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: trec
-doc_to_text: 'Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}'
+doc_to_text: "Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_classification_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: ["\n"]
 metric_list:
   - metric: "classification_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/trec_e.yaml

@@ -5,17 +5,17 @@ task: longbench_trec_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: trec_e
-doc_to_text: 'Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}'
+doc_to_text: "Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_classification_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: ["\n"]
 metric_list:
   - metric: "classification_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/triviaqa.yaml

@@ -5,17 +5,17 @@ task: longbench_triviaqa
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: triviaqa
-doc_to_text: 'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}'
+doc_to_text: "Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: ["\n"]
 metric_list:
   - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/triviaqa_e.yaml

@@ -5,17 +5,17 @@ task: longbench_triviaqa_e
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: triviaqa_e
-doc_to_text: 'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}'
+doc_to_text: "Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}"
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: ["\n"]
 metric_list:
   - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/longbench/vcsum.yaml

@@ -5,17 +5,17 @@ task: longbench_vcsum
 dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: vcsum
-doc_to_text: '下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{{context}}\n\n会议总结：'
+doc_to_text: "下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{{context}}\n\n会议总结："
 doc_to_target: '{{answers}}'
 process_results: !function metrics.get_rouge_zh_score
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
-  do_sample: True
+  do_sample: False
   until: []
 metric_list:
   - metric: "rouge_zh_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/minerva_math/README.md

 # MATH
+
 ℹ️ This is the 4-shot variant!
+
 ## Paper
+
 Measuring Mathematical Problem Solving With the MATH Dataset
 https://arxiv.org/abs/2103.03874
 
-Many intellectual endeavors require mathematical problem solving, but this skill remains beyond the capabilities of computers. To measure this ability in machine learning models, we introduce MATH, a new dataset of 12,500 challenging competition mathematics problems. Each problem in MATH has a full step-by-step solution which can be used to teach models to generate answer derivations and explanations.
+Many intellectual endeavors require mathematical problem solving, but this skill remains beyond the capabilities of
+computers. To measure this ability in machine learning models, we introduce MATH, a new dataset of 12,500 challenging
+competition mathematics problems. Each problem in MATH has a full step-by-step solution which can be used to teach
+models to generate answer derivations and explanations.
 
-NOTE: The few-shot and the generated answer extraction is based on the [Minerva](https://arxiv.org/abs/2206.14858) and exact match equivalence is calculated using the `sympy` library. This requires additional dependencies, which can be installed via the `lm-eval[math]` extra.
+NOTE: The few-shot and the generated answer extraction is based on the [Minerva](https://arxiv.org/abs/2206.14858) and
+exact match equivalence is calculated using the `sympy` library. This requires additional dependencies, which can be
+installed via the `lm-eval[math]` extra.
 
 Homepage: https://github.com/hendrycks/math
 
-
 ## Citation
+
 ```
 @article{hendrycksmath2021,
   title={Measuring Mathematical Problem Solving With the MATH Dataset},
@@ -49,13 +57,18 @@ Eprint = {arXiv:2206.14858},
 The checklist is the following:
 
 For adding novel benchmarks/datasets to the library:
-* [x] Is the task an existing benchmark in the literature?
-  * [x] Have you referenced the original paper that introduced the task?
-  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
-    * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical.
 
+* [x] Is the task an existing benchmark in the literature?
+    * [x] Have you referenced the original paper that introduced the task?
+    * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the
+      reference implementation and documented how to run such a test?
+        * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have
+          a few-shot evaluation for GPT-3, however the few-shot context used here is sourced
+          from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is
+          comparable to that provided in the paper, though not identical.
 
 If other tasks on this dataset are already supported:
+
 * [x] Is the "Main" variant of this task clearly denoted?
 * [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
 * [x] Have you noted which, if any, published evaluation setups are matched by this variant?
@@ -65,4 +78,7 @@ If other tasks on this dataset are already supported:
 - [ ] zero-shot variant
 
 ### Changelog
-version 2.0: (21-Feb-2025); added math_verify (extraction) metric. For details [see](https://huggingface.co/blog/math_verify_leaderboard)
+
+- version 2.0: (21-Feb-2025); added math_verify (extraction) metric. For
+  details [see](https://huggingface.co/blog/math_verify_leaderboard)
+- version 3.0 (21-Aug-2025); pass the full solution and model generation to `math_verify`'s `parse`

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -24,7 +24,7 @@ metric_list:
     higher_is_better: true
 num_fewshot: 4
 metadata:
-  version: 2.0
+  version: 3.0
 fewshot_config:
   sampler: first_n
   samples: !function utils.list_fewshot_samples