Add loncxt tasks (#2629)

suport for longcontext (and other synthetic tasks)
* add ruler
* add longbench
* pass `metadata` to TaskConfig

lm_eval/tasks/longbench/metrics.py

+# MIT License
+#
+# Copyright (c) 2023 THU-KEG & Zhipu AI
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import re
+import string
+from collections import Counter
+
+try:
+    import jieba
+    from fuzzywuzzy import fuzz
+    from rouge import Rouge
+except ImportError:
+    raise ImportError(
+        'Please install the required dependencies for this task with `pip install lm_eval["longbench"] or `pip install jeiba fuzzywuzzy rouge`'
+    )
+
+# taken from https://github.com/THUDM/LongBench
+
+
+def normalize_answer(s: str) -> str:
+    """Lower text and remove punctuation, articles and extra whitespace."""
+
+    def remove_articles(text):
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+
+    def white_space_fix(text):
+        return " ".join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def normalize_zh_answer(s: str) -> str:
+    """Lower text and remove punctuation, extra whitespace."""
+
+    def white_space_fix(text):
+        return "".join(text.split())
+
+    def remove_punc(text):
+        cn_punctuation = "！？｡。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."
+        all_punctuation = set(string.punctuation + cn_punctuation)
+        return "".join(ch for ch in text if ch not in all_punctuation)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_punc(lower(s)))
+
+
+def count_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    numbers = re.findall(r"\d+", prediction)
+    right_num = 0
+    for number in numbers:
+        if str(number) == str(ground_truth):
+            right_num += 1
+    final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+    return float(final_score)
+
+
+def retrieval_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    pattern = r"Paragraph (\d+)"
+    matches = re.findall(pattern, ground_truth)
+    ground_truth_id = matches[0]
+    numbers = re.findall(r"\d+", prediction)
+    right_num = 0
+    for number in numbers:
+        if str(number) == str(ground_truth_id):
+            right_num += 1
+    final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+    return float(final_score)
+
+
+def retrieval_zh_score(
+    predictions: list[str], references: list[str], **kwargs
+) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    pattern = r"段落(\d+)"
+    matches = re.findall(pattern, ground_truth)
+    ground_truth_id = matches[0]
+    numbers = re.findall(r"\d+", prediction)
+    right_num = 0
+    for number in numbers:
+        if str(number) == str(ground_truth_id):
+            right_num += 1
+    final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+    return float(final_score)
+
+
+def code_sim_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    all_lines = prediction.lstrip("\n").split("\n")
+    prediction = ""
+    for line in all_lines:
+        if ("`" not in line) and ("#" not in line) and ("//" not in line):
+            prediction = line
+            break
+    return fuzz.ratio(prediction, ground_truth) / 100
+
+
+def classification_score(
+    predictions: list[str], references: list[str], **kwargs
+) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    em_match_list = []
+    all_classes = kwargs["all_classes"]
+    for class_name in all_classes:
+        if class_name in prediction:
+            em_match_list.append(class_name)
+    for match_term in em_match_list:
+        if match_term in ground_truth and match_term != ground_truth:
+            em_match_list.remove(match_term)
+    if ground_truth in em_match_list:
+        score = 1.0 / len(em_match_list)
+    else:
+        score = 0.0
+    return score
+
+
+def rouge_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    rouge = Rouge()
+    try:
+        scores = rouge.get_scores([prediction], [ground_truth], avg=True)
+        # ruff: noqa
+    except:
+        return 0.0
+    return scores["rouge-l"]["f"]
+
+
+def rouge_zh_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    prediction = " ".join(list(jieba.cut(prediction, cut_all=False)))
+    ground_truth = " ".join(list(jieba.cut(ground_truth, cut_all=False)))
+    score = rouge_score([prediction], [ground_truth])
+    return score
+
+
+def f1_score(predictions: list[str], references: list[str], **kwargs):
+    try:
+        prediction, ground_truth = predictions[0], references[0]
+    except:
+        return 0.0
+    common = Counter(prediction) & Counter(ground_truth)
+    num_same = sum(common.values())
+    if num_same == 0:
+        return 0
+    precision = 1.0 * num_same / len(prediction)
+    recall = 1.0 * num_same / len(ground_truth)
+    f1 = (2 * precision * recall) / (precision + recall)
+    return f1
+
+
+def qa_f1_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    normalized_prediction = normalize_answer(prediction)
+    normalized_ground_truth = normalize_answer(ground_truth)
+
+    prediction_tokens = normalized_prediction.split()
+    ground_truth_tokens = normalized_ground_truth.split()
+    try:
+        res = f1_score(prediction_tokens, ground_truth_tokens)
+    except:
+        return 0.0
+    return res
+
+
+def qa_f1_zh_score(predictions: list[str], references: list[str], **kwargs) -> float:
+    prediction, ground_truth = predictions[0], references[0]
+    prediction_tokens = list(jieba.cut(prediction, cut_all=False))
+    ground_truth_tokens = list(jieba.cut(ground_truth, cut_all=False))
+    prediction_tokens = [normalize_zh_answer(token) for token in prediction_tokens]
+    ground_truth_tokens = [normalize_zh_answer(token) for token in ground_truth_tokens]
+    prediction_tokens = [token for token in prediction_tokens if len(token) > 0]
+    ground_truth_tokens = [token for token in ground_truth_tokens if len(token) > 0]
+    return f1_score(prediction_tokens, ground_truth_tokens)

lm_eval/tasks/longbench/multi_news.yaml

+
+tag:
+  - longbench
+task: longbench_multi_news
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: multi_news
+doc_to_text: 'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{{context}}\n\nNow, write a one-page summary of all the news.\n\nSummary:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 512
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.rouge_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/multi_news_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_multi_news_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: multi_news_e
+doc_to_text: 'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{{context}}\n\nNow, write a one-page summary of all the news.\n\nSummary:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 512
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.rouge_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/multifieldqa_en.yaml

+
+tag:
+  - longbench
+task: longbench_multifieldqa_en
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: multifieldqa_en
+doc_to_text: 'Read the following text and answer briefly.\n\n{{context}}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 64
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/multifieldqa_en_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_multifieldqa_en_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: multifieldqa_en_e
+doc_to_text: 'Read the following text and answer briefly.\n\n{{context}}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 64
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/multifieldqa_zh.yaml

+
+tag:
+  - longbench
+task: longbench_multifieldqa_zh
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: multifieldqa_zh
+doc_to_text: '阅读以下文字并用中文简短回答：\n\n{{context}}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{{input}}\n回答：'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 64
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_zh_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/musique.yaml

+
+tag:
+  - longbench
+task: longbench_musique
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: musique
+doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 32
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/narrativeqa.yaml

+
+tag:
+  - longbench
+task: longbench_narrativeqa
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: narrativeqa
+doc_to_text: 'You are given a story, which can be either a novel or a movie script, and a question. Answer the question asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {{context}}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 128
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/passage_count.yaml

+
+tag:
+  - longbench
+task: longbench_passage_count
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: passage_count
+doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 32
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.count_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/passage_count_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_passage_count_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: passage_count_e
+doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 32
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.count_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/passage_retrieval_en.yaml

+
+tag:
+  - longbench
+task: longbench_passage_retrieval_en
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: passage_retrieval_en
+doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: '
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 32
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.retrieval_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/passage_retrieval_en_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_passage_retrieval_en_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: passage_retrieval_en_e
+doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: '
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 32
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.retrieval_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/passage_retrieval_zh.yaml

+
+tag:
+  - longbench
+task: longbench_passage_retrieval_zh
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: passage_retrieval_zh
+doc_to_text: '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{{context}}\n\n下面是一个摘要\n\n{{input}}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 32
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.retrieval_zh_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/qasper.yaml

+
+tag:
+  - longbench
+task: longbench_qasper
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: qasper
+doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 128
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/qasper_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_qasper_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: qasper_e
+doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 128
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.qa_f1_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/qmsum.yaml

+
+tag:
+  - longbench
+task: longbench_qmsum
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: qmsum
+doc_to_text: 'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{{context}}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {{input}}\nAnswer:'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 512
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.rouge_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/repobench-p.yaml

+
+tag:
+  - longbench
+task: longbench_repobench-p
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: repobench-p
+doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 64
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.code_sim_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/repobench-p_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_repobench-p_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: repobench-p_e
+doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 64
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.code_sim_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/samsum.yaml

+
+tag:
+  - longbench
+task: longbench_samsum
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: samsum
+doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 128
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.rouge_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0

lm_eval/tasks/longbench/samsum_e.yaml

+
+tag:
+  - longbench_e
+task: longbench_samsum_e
+dataset_path: THUDM/LongBench
+test_split: test
+dataset_name: samsum_e
+doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}'
+doc_to_target: '{{answers}}'
+generation_kwargs:
+  max_gen_toks: 128
+  temperature: 1
+  do_sample: True
+metric_list:
+  - metric: !function metrics.rouge_score
+    aggregation: mean
+    higher_is_better: True
+metadata:
+  version: 1.0