[longbench] fix metric calculation (#2983)

* use all answers

* use middle truncation

* maybe fix classification score

* strip classification preds

* [vllm] remove stop tokens post-hoc

* strip all preds

* pacify pre-commit

* start on truncation utility

* add to readme

* add a footgun doc

* fix newline in yaml templates

* do not strip code_sim preds!

* fix pre-commit config

* fix instruction warning

* add not to longbench readme

lm_eval/tasks/longbench/multifieldqa_en.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: multifieldqa_en
 doc_to_text: 'Read the following text and answer briefly.\n\n{{context}}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/multifieldqa_en_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: multifieldqa_en_e
 doc_to_text: 'Read the following text and answer briefly.\n\n{{context}}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/multifieldqa_zh.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: multifieldqa_zh
 doc_to_text: '阅读以下文字并用中文简短回答：\n\n{{context}}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{{input}}\n回答：'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_zh_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_zh_score
+  - metric: "qa_f1_zh_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/musique.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: musique
 doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/narrativeqa.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: narrativeqa
 doc_to_text: 'You are given a story, which can be either a novel or a movie script, and a question. Answer the question asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {{context}}\n\nNow, answer the question based on the story asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/passage_count.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_count
 doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_count_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.count_score
+  - metric: "count_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/passage_count_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_count_e
 doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_count_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.count_score
+  - metric: "count_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/passage_retrieval_en.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_retrieval_en
 doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: '
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_retrieval_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.retrieval_score
+  - metric: "retrieval_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/passage_retrieval_en_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_retrieval_en_e
 doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: '
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_retrieval_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.retrieval_score
+  - metric: "retrieval_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/passage_retrieval_zh.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: passage_retrieval_zh
 doc_to_text: '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{{context}}\n\n下面是一个摘要\n\n{{input}}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_retrieval_zh_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.retrieval_zh_score
+  - metric: "retrieval_zh_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/qasper.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: qasper
 doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/qasper_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: qasper_e
 doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/qmsum.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: qmsum
 doc_to_text: 'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{{context}}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {{input}}\nAnswer:'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_rouge_score
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.rouge_score
+  - metric: "rouge_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/repobench-p.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: repobench-p
 doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_code_sim_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.code_sim_score
+  - metric: "code_sim_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/repobench-p_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: repobench-p_e
 doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_code_sim_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
   until: []
 metric_list:
-  - metric: !function metrics.code_sim_score
+  - metric: "code_sim_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/samsum.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: samsum
 doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_rouge_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
   do_sample: True
-  until: ['\n']
+  until: ["\n"]
 metric_list:
-  - metric: !function metrics.rouge_score
+  - metric: "rouge_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/samsum_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: samsum_e
 doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_rouge_score
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
   do_sample: True
-  until: ['\n']
+  until: ["\n"]
 metric_list:
-  - metric: !function metrics.rouge_score
+  - metric: "rouge_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/trec.yaml

@@ -6,16 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: trec
 doc_to_text: 'Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}'
-doc_to_target: '{{answers[0]}}'
-process_results: !function metrics.classification_score
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_classification_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
-  until: ['\n']
+  until: ["\n"]
 metric_list:
   - metric: "classification_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/trec_e.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: trec_e
 doc_to_text: 'Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_classification_score
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
   do_sample: True
-  until: ['\n']
+  until: ["\n"]
 metric_list:
-  - metric: !function metrics.classification_score
+  - metric: "classification_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/longbench/triviaqa.yaml

@@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: triviaqa
 doc_to_text: 'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}'
-doc_to_target: '{{answers[0]}}'
+doc_to_target: '{{answers}}'
+process_results: !function metrics.get_qa_f1_score
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
   do_sample: True
-  until: ['\n']
+  until: ["\n"]
 metric_list:
-  - metric: !function metrics.qa_f1_score
+  - metric: "qa_f1_score"
     aggregation: mean
     higher_is_better: True
 metadata:
-  version: 2.0
+  version: 3.0