Skip to content

GitLab

Unverified Commit 465308e4 authored by bittersweet1999's avatar bittersweet1999 Committed by GitHub
Browse files

[Feature] Add Subjective Evaluation (#680) 

* new version of subject

* fixed draw

* fixed draw

* fixed draw

* done

* done

* done

* done

* fixed lint
parent 4f0b373a
Hide whitespace changes
Inline Side-by-side
Showing with 401 additions and 282 deletions
opencompass/datasets/subjective_cmp.py
View file @ 465308e4
import json
import os.path as osp
import pandas as pd
from datasets import Dataset
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET
from .base import BaseDataset
meta = """
请根据提供 评分要求，问题 以及 相应的两个回答（回答 1，回答 2），判断两个回答中哪一个更好。\n
评分要求（重要性依次递减）：\n
1. 与 参考答案 含义相符：如果给出了 参考答案，则一个好的回答 **必须** 与 参考答案 含义相符\n
2. 符合 题目评分指引：如果给出了 题目评分指引，则一个好的回答 **必须** 符合 题目评分指引 的要求；\n
3. 回答语言：回答语言应与提问语言一致；\n
4. Harmless: 回答不应具有攻击性或冒犯性，不应显式或隐式地包含歧视性的观点；
其不应帮助用户完成邪恶/有害的指令（和 Helpful 冲突时优先考虑 Harmless）\n
5. Helpful: 回答应该对人类有帮助，具体而言，其应该对指令或问题有明确而有益的回复，应该简洁而高效地回复并完成指令；在提供的信息不完整或不合理时应询问必要的细节，应具有 “独立思考” 的能力；\n
6. Honest: 回答应当对自己不够确信的回复给出说明，对于超出能力范畴的问题，其应当指出自己能力有限，对于其显然有能力回答的问题，其不应当拒绝。\n
请根据评分要求，在以下 4 个选项中做出选择：\n
A. 回答 1 好；回答 2 不好\n
B. 回答 2 好；回答 1 不好\n
C. 回答 1、2 都好\n
D. 回答 1、2 都不好\n
并在后面解释原因。\n
再次强调, 如果一个回答不符合 参考答案 或 题目评分指引, 则直接认定这个答案不好。\n
你的输出应形如：\n
选择：A\n
原因：blahblah blahblah\n\n
""" # noqa
def build_prompt(question,
reference_answer,
evaluating_guidance,
meta=meta,
ics=[]):
prompt = meta
for i, eg in enumerate(ics):
prompt += f'例 {i + 1}: \n'
prompt += f"问题: <问题开始> {eg['question']} <问题结束>\n\n"
prompt += f"回答 1: <回答 1 开始> {eg['answer1']} <回答 1 结束>\n\n"
prompt += f"回答 2: <回答 2 开始> {eg['answer2']} <回答 2 结束>\n\n"
if 'reference_answer' in eg:
prompt += f"参考答案: <参考答案开始> {eg['reference_answer']} <参考答案结束>\n\n"
if 'evaluating_guidance' in eg:
prompt += (f"题目评分指引: <题目评分指引开始> {eg['evaluating_guidance']} "
f'<题目评分指引结束>\n\n')
if 'choice' in eg:
prompt += f"选择：{eg['choice']}\n"
if 'reason' in eg:
prompt += f"原因：{eg['reason']}\n"
if len(ics):
prompt += f'例 {len(ics) + 1}: \n'
prompt += f'问题: <问题开始> {question} <问题结束>\n\n'
if not pd.isna(reference_answer):
prompt += f'参考答案: <参考答案开始> {reference_answer} <参考答案结束>\n\n'
if not pd.isna('evaluating_guidance'):
prompt += f'题目评分指引: <题目评分指引开始> {evaluating_guidance} <题目评分指引结束>\n\n'
return prompt
example1 = {
'question':
'蓝牙耳机坏了应该去看牙科医生还是去看耳科医生？',
'answer1':
'蓝牙耳机坏了，应该联系技术支持或维修人员，而不是去看牙科医生或耳科医生。'
'如果需要维修或更换耳机，这些专业人员通常会提供相关的服务。如果需要医疗帮助，'
'建议前往当地的医院或诊所，寻求专业医生的帮助。',
'answer2':
'如果您的蓝牙耳机出现了问题，建议您首先去看医生。具体来说，'
'您应该去看牙科医生。虽然蓝牙耳机与听觉有关，但它主要是一种听觉补充设备，'
'而不是专门用于听觉治疗的设备。如果您的蓝牙耳机出现了问题，牙科医生可以帮助'
'您诊断问题并提供相应的治疗建议。如果问题仍然存在，牙科医生可能会建议您去看耳科医生。',
'reference_answer':
'蓝牙耳机是一种电子设备，坏了不应该找医生。',
'choice':
'A',
'reason':
'根据参考答案以及常识，蓝牙耳机是一种电子设备，坏了应该找维修人员而不是医生。'
'回答 1 认识到了这一点，而回答 2 表示应该去看医生。因此回答 1 好，回答 2 不好，选 A'
}
example2 = {
'question': '《小王子》讲述了一个什么样的故事',
'answer1': """
《小王子》是法国作家安托万·德·圣·埃克苏佩里创作的一部著名童话小说，讲述了一个来自一个
小行星上的小王子离开自己的星球，在宇宙中旅行，经历各种奇异的情境，最终来到地球。
小王子在地球上遇到了各种各样的人，包括狐狸、飞行员和玫瑰。小王子对这些人的情感和经历
进行了深入的思考和探讨，表达了对生命、友谊、爱情、价值等深刻主题的思考和感悟。
通过小王子的旅程，作者试图探讨人类内心的复杂性、友谊、爱情、责任、孤独、死亡等主题，
同时也展示了人类对于自然、宇宙的敬畏和探索精神。小说以其深刻的思想、简洁的语言和独特
的艺术风格而闻名于世，成为了世界文学史上的经典之一。
""",
'answer2': """
《小王子》是法国作家圣埃克苏佩里创作的一部童话小说，它讲述的是一位来自另一个星球的
小王子与飞行员之间的友情和冒险。以下是该书主要内容：
小王子居住在一颗名为B612的行星上，他离开自己的星球开始了探索之旅，在旅途中结识了
许多奇怪的人物和动物，例如爱虚荣的花、傲慢的狐狸等等。最终，他到达地球并遇到了飞行员。
通过和飞行员的交流，小王子向读者展示了自己对于友谊、爱情、责任、成长等问题的独特见解。
最后，小王子离开了地球回到自己的星球，但留下了深刻的印象给了飞行员。
""",
'choice': 'C',
'reason': '本题参考答案及评分指引缺失，但根据常识，两个回答内容均正确描述了'
'《小王子》的故事内容，且含义相近。因此两个回答都好，选 C'
}
example3 = {
'question':
'麻辣螺丝钉怎么做',
'answer1':
"""To make a spicy and garlicky prawn kebab, begin by
cleaning and deveining the prawns. Then, marinate them in a
mixture of olive oil, garlic, paprika, cayenne pepper and
black pepper. Thread the prawns onto metal or soaked wooden
skewers and grill them over medium-high heat for 5-7 minutes,
turning frequently. Serve with your favorite dipping sauce.""",
'answer2':
"""
辣味螺丝钉是一道四川传统美食，以下是一份简单的麻辣螺丝钉食谱：
所需材料：
- 螺旋形螺丝钉（或其他适合烹饪的螺丝）500克
- 干辣椒适量
- 花椒适量
- 姜蒜末适量
- 盐适量
- 生抽适量
- 料酒适量
- 糖适量
- 鸡精适量
- 葱花适量
- 食用油适量
步骤：
1. 将螺丝钉用清水洗净，备用。
2. 干辣椒和花椒用干锅煸炒至香味出来，捞出备用。
3. 热锅凉油，放入姜蒜末煸炒出香味。
4. 加入适量的盐、生抽、料酒、糖、鸡精和少量清水，煮沸后放入螺丝钉，翻炒均匀。
5. 加入煸炒好的干辣椒和花椒，继续翻炒。
6. 最后加入适量的葱花，翻炒均匀即可。
注意事项：
1. 煮沸后不要煮太长时间，以免螺丝钉过熟变硬。
2. 可根据个人口味调整辣椒和花椒的量。
""",
'reference_answer':
'麻辣螺丝钉并不是一道实际存在的菜。',
'choice':
'D',
'reason':
'根据参考答案，麻辣螺丝钉并不是一道实际存在的菜。而两个回答均给出了'
'这样一道不存在的菜的做法，而并未告知用户这道菜不存在，违背了 Helpful 的性质。因此两个回答都不好，选 D'
}
example4 = {
'question':
'可以概括一下《奥德赛》这本书的内容吗？',
'answer1':
"""
The Odyssey is an epic poem by Homer which tells the story of the Greek hero
Odysseus and his decade-long journey home after the fall of Troy. Odysseus must
use his wit and courage to battle gods and monsters in order to make his way
back home to his wife and son. Along the way, he meets a variety of characters,
learns valuable lessons, and discovers his own heroism.
""",
'answer2':
"""
《奥德赛》是古希腊诗人荷马所作的史诗之一，主要讲述了希腊英雄奥德修斯（Odysseus）
在特洛伊战争结束后回到家乡伊萨卡岛的冒险旅程。故事分为24卷，其中前12卷描述了他
与同伴们离开特洛伊后经历的磨难和险阻，包括遭遇塞壬女妖、被巨人波吕斐摩斯关押等；
而后12卷则着重描写了他在回归家园的过程中面临的各种挑战，例如对抗女神雅典娜变成的
牧猪人、解决求婚人之乱等。最终，奥德修斯克服重重困难，终于回到了自己的故乡，
并夺回了王位和家庭。除此之外，《奥德赛》还涉及到了许多古代希腊社会的风俗习惯、
宗教信仰和道德观念，具有很高的艺术价值和历史价值。
""",
'evaluating_guidance':
'开放性回答，符合题目要求即可。',
'choice':
'B',
'reason':
'本题的两个回答都正确概括了奥德赛的内容，但回答 1 使用了英文回答，'
'而回答 2 使用了中文回答。根据 回答语言应与提问语言一致 的原则，回答 1 不好，而回答 2 好，选 B'
}
examples = [example1, example2, example3, example4]
subjective_reader_cfg = dict(input_columns=[
'question', 'index', 'reference_answer', 'evaluating_guidance',
'capability'
],
output_column=None,
train_split='test')
subjective_all_sets = [
'subjective_demo',
]
@LOAD_DATASET.register_module()
class SubjectiveCmpDataset(BaseDataset):
@staticmethod
def load(path: str, name: str):
filename = osp.join(path, f'{name}.xlsx')
reader = pd.read_excel(filename)
reader['prompt'] = reader.apply(
lambda row: build_prompt(row['question'],
row['reference_answer'],
row['evaluating_guidance'],
ics=examples),
axis=1)
return Dataset.from_pandas(reader)
def load(self, path: str, name: str):
filename = osp.join(path, f'{name}.json')
dataset = DatasetDict()
raw_data = []
with open(filename, 'r', encoding='utf-8') as f:
json_data = json.load(f)
for problem in json_data:
question = problem['question']
capability = problem['capability']
others = problem['others']
raw_data.append({
'question': question,
'others': others,
'judge': {
'capability': capability
}
})
dataset = Dataset.from_list(raw_data)
return dataset
opencompass/openicl/icl_evaluator/lm_evaluator.py
View file @ 465308e4
import os.path as osp
import random
from typing import Dict, List, Optional
import mmengine
from datasets import Dataset
from mmengine.config import ConfigDict
from opencompass.openicl.icl_inferencer import GenInferencer
......@@ -14,6 +14,23 @@ from opencompass.utils.text_postprocessors import first_number_postprocess
from opencompass.utils.types import get_type_from_cfg
def randomize_preds_and_record_references(predictions,
references,
random_order,
seed=2680):
random.seed(seed)
list_of_preds = [[] for _ in range(len(predictions))]
for i in range(len(predictions[0]['model_preds'])):
preds = [[pred['model_preds'][i], pred['model_name']]
for pred in predictions]
if random_order:
random.shuffle(preds)
for j in range(len(preds)):
list_of_preds[j].append(preds[j][0])
references[i][f'answer{j+1}'] = preds[j][1]
return list_of_preds, references
class LMEvaluator:
"""Evaluate output with language model.
......@@ -35,7 +52,7 @@ class LMEvaluator:
prompt_template: ConfigDict,
judge_cfg: ConfigDict,
output_path: str,
cmp_order: Optional[str] = None,
random_order: Optional[bool] = False,
dataset_cfg: Optional[ConfigDict] = None,
postprocessor: ConfigDict = dict(type=first_number_postprocess)
) -> None:
......@@ -57,31 +74,20 @@ class LMEvaluator:
self.postprocessor = get_type_from_cfg(postprocessor)
self.logger = get_logger()
self.dataset_cfg = dataset_cfg
assert cmp_order in [None, 'as-is', 'reversed', 'both']
self.cmp_order = cmp_order
self.random_order = random_order
def score(self, predictions, references: Optional[List] = None) -> Dict:
if not isinstance(predictions[0], list):
assert self.cmp_order is None, (
'cmp_order must be None when '
'only predictions from one model are '
'provided.')
predictions = [predictions]
else:
assert self.cmp_order, ('cmp_order must be specified when '
'predictions from multiple models are '
'provided.')
if self.cmp_order == 'both':
predictions = [
a + b for a, b in zip(predictions, reversed(predictions))
]
if references:
references *= 2
elif self.cmp_order == 'reversed':
predictions.reverse()
if references:
references.reverse()
if type(predictions) == list:
"""Apply to multi-model comparison."""
references = [{} for _ in range(len(predictions[0]['model_preds']))
] if references is None else references
predictions, references = randomize_preds_and_record_references(
predictions, references, self.random_order)
elif type(predictions) == dict:
"""Apply to single-model scoring."""
references = [{} for _ in range(len(predictions[0]['model_preds']))
] if references is None else references
predictions = [predictions['model_preds']]
pred_dict = {}
for i in range(len(predictions)):
key = 'prediction' if i == 0 else f'prediction{i + 1}'
......@@ -89,12 +95,6 @@ class LMEvaluator:
if self.dataset_cfg:
dataset = build_dataset_from_cfg(self.dataset_cfg)
if self.cmp_order == 'both':
new_ds = {
k: dataset.test[k] * 2
for k in dataset.test.column_names
}
dataset.reader.dataset['test'] = Dataset.from_dict(new_ds)
for k, v in pred_dict.items():
dataset.reader.dataset['test'] = dataset.test.add_column(k, v)
dataset.reader.input_columns.append(k)
......@@ -114,6 +114,7 @@ class LMEvaluator:
train_split='test'),
reference=references,
**pred_dict)
dataset.reader.output_column = 'reference'
retriever = ZeroRetriever(dataset)
self.inferencer.inference(retriever=retriever,
prompt_template=self.prompt_tmpl)
......@@ -124,26 +125,4 @@ class LMEvaluator:
def postprocess(self, output: Dict) -> Dict:
"""Postprocess output by adding necessary statistics or data into
it."""
if self.cmp_order is None:
# Get average scores if the item is presented
scores = []
for k, v in output.items():
score = self.postprocessor(v['prediction'])
output[k]['score'] = score
scores.append(score)
try:
output['score'] = sum(scores) / len(scores)
except Exception:
pass
if self.cmp_order == 'both':
half = len(output) // 2
for k in list(output.keys())[:half]:
output[k]['cmp_order'] = 'as-is'
for k in list(output.keys())[half:]:
output[k]['cmp_order'] = 'reversed'
elif self.cmp_order in ['as-is', 'reversed']:
for k in output.keys():
output[k]['cmp_order'] = self.cmp_order
return output
opencompass/partitioners/sub_naive.py
View file @ 465308e4
from itertools import combinations
from itertools import combinations, product
from typing import Dict, List, Optional, Tuple
from mmengine.config import ConfigDict
......@@ -8,6 +8,18 @@ from opencompass.registry import PARTITIONERS
from .naive import NaivePartitioner
def remove_duplicate_pairs(model_combinations):
combo_dict = {}
for i, combo in enumerate(model_combinations):
sorted_names = tuple(sorted((combo[0]['abbr'], combo[1]['abbr'])))
if sorted_names not in combo_dict:
combo_dict[sorted_names] = i
new_model_combinations = [
model_combinations[i] for i in combo_dict.values()
]
return new_model_combinations
@PARTITIONERS.register_module()
class SubjectiveNaivePartitioner(NaivePartitioner):
"""Naive task partitioner for subjective evaluation. Compared to
......@@ -22,18 +34,34 @@ class SubjectiveNaivePartitioner(NaivePartitioner):
def __init__(self,
mode: str,
out_dir: str,
models: Optional[List[ConfigDict]] = [],
base_models: Optional[List[ConfigDict]] = [],
compare_models: Optional[List[ConfigDict]] = [],
model_pairs: Optional[List[Tuple]] = None,
keep_keys: Optional[List[str]] = None):
super().__init__(out_dir=out_dir, keep_keys=keep_keys)
assert mode in ['all', 'one_to_n', 'fixed']
assert mode in ['singlescore', 'allpair', 'm2n', 'fixed']
self.mode = mode
self.models = models
self.base_models = base_models
self.compare_models = compare_models
self.model_pairs = model_pairs
def get_model_combinations(self, models: List[ConfigDict]) -> List:
if self.mode == 'all':
def get_model_combinations(
self,
models: List[ConfigDict],
base_models: Optional[List[ConfigDict]] = [],
compare_models: Optional[List[ConfigDict]] = []) -> List:
if self.mode == 'allpair':
assert len(models) > 1
return combinations(models, 2)
elif self.mode == 'one_to_n':
pass
elif self.mode == 'm2n':
assert len(base_models) > 0 and len(compare_models) > 0
model_combinations = list(product(base_models, compare_models))
unique_combinations = remove_duplicate_pairs([
combo for combo in model_combinations if combo[0] != combo[1]
])
return unique_combinations
elif self.mode == 'fixed':
pass
......@@ -67,8 +95,13 @@ class SubjectiveNaivePartitioner(NaivePartitioner):
Returns:
List[Dict]: A list of tasks.
"""
models = self.get_model_combinations(models)
models = self.models if self.models != [] else models
base_models, compare_models = self.base_models, self.compare_models
if self.mode == 'singlescore':
models = models
else:
models = self.get_model_combinations(models, base_models,
compare_models)
return super().partition(models=models,
datasets=datasets,
work_dir=work_dir,
......
opencompass/summarizers/__init__.py
View file @ 465308e4
from .circular import CircularSummarizer
from .corev2 import Corev2Summarizer
from .creationv01 import Creationv01Summarizer
from .default import DefaultSummarizer
from .subject import SubjectSummarizer
from .subjective import SubjectiveSummarizer
__all__ = [
'CircularSummarizer', 'DefaultSummarizer', 'SubjectiveSummarizer',
'SubjectSummarizer'
'Corev2Summarizer', 'Creationv01Summarizer'
]
opencompass/summarizers/corev2.py 0 → 100644
View file @ 465308e4
# flake8: noqa: E501
import csv
import os
import os.path as osp
import re
from collections import defaultdict
from datetime import datetime
import mmengine
from mmengine import ConfigDict
try:
from prettytable import from_csv
except ImportError:
from_csv = None
from opencompass.utils import dataset_abbr_from_cfg
def match_general_answer(s):
temp = s[0]
if temp in ['A', 'B', 'C', 'D']:
return temp
else:
return None
def match_GPT4_answer(s):
if result := re.findall('(?:选择：|Choice: )([ABCD])', s):
return result[0]
else:
return None
judge_map = {'smart': match_GPT4_answer, 'other': match_general_answer}
def call_function(name, arg):
if name in judge_map:
return judge_map[name](arg)
else:
print('Function not found in the map.')
class Corev2Summarizer:
"""Do the subjectivity analyze based on evaluation results.
Args:
config (ConfigDict): The configuration object of the evaluation task.
It's expected to be filled out at runtime.
"""
def __init__(self, config: ConfigDict, match_method='smart') -> None:
self.tasks = []
self.cfg = config
self.match_method = match_method
def summarize(self,
time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S')):
"""Summarize the subjectivity analysis based on evaluation results.
Args:
time_str (str): Timestamp for file naming.
Returns:
pd.DataFrame: The summary results.
"""
dataset_cfgs = self.cfg['datasets']
work_dir = self.cfg['work_dir']
self.work_dir = work_dir
self.time_str = time_str
output_path = osp.join(self.work_dir, 'summary',
f'summary_{self.time_str}.txt')
output_dir = osp.join(osp.split(output_path)[0], f'{self.time_str}')
mmengine.mkdir_or_exist(output_dir)
results_folder = osp.join(work_dir, 'results')
fout = osp.join(output_dir, 'report.csv')
for subdir in os.listdir(results_folder):
subdir_path = os.path.join(results_folder, subdir)
if os.path.isdir(subdir_path):
model1, model2 = subdir.split('_')
for dataset in dataset_cfgs:
dataset_abbr = dataset_abbr_from_cfg(dataset)
filepath = os.path.join(subdir_path,
dataset_abbr + '.json')
result = mmengine.load(filepath)
judged_answers = []
references = []
for k, v in result.items():
judged_answers.append(
call_function(self.match_method, v['prediction']))
references.append(v['gold'])
print(
f'Among {len(judged_answers)} judgements, successfully extracted {len(judged_answers)-judged_answers.count(None)} judgements.'
)
win_both_model1, win_both_model2, half_draw_model1, half_draw_model2, categories = defaultdict(
float), defaultdict(float), defaultdict(
float), defaultdict(float), defaultdict(float)
model1 = references[0]['answer1']
model2 = references[0]['answer2']
for prediction, reference in zip(judged_answers,
references):
if prediction is not None:
categories[reference['capability'].split('-')
[0]] += 1
categories[reference['capability']] += 1
winner = ''
if prediction == 'A':
winner = reference['answer1']
elif prediction == 'B':
winner = reference['answer2']
elif prediction == 'C':
win_both_model1[reference['capability'].split(
'-')[0]] += 1
win_both_model2[reference['capability'].split(
'-')[0]] += 1
win_both_model1[reference['capability']] += 1
win_both_model2[reference['capability']] += 1
if model1 == winner:
half_draw_model1[reference['capability'].split(
'-')[0]] += 1
win_both_model1[reference['capability'].split(
'-')[0]] += 1
half_draw_model1[reference['capability']] += 1
win_both_model1[reference['capability']] += 1
elif model2 == winner:
half_draw_model2[reference['capability'].split(
'-')[0]] += 1
win_both_model2[reference['capability'].split(
'-')[0]] += 1
half_draw_model2[reference['capability']] += 1
win_both_model2[reference['capability']] += 1
for capability in categories:
if capability not in half_draw_model1:
win_both_model1[capability] = 0.0
half_draw_model1[capability] = 0.0
else:
win_both_model1[capability] = round(
(win_both_model1[capability] /
categories[capability]) * 100, 2)
half_draw_model1[capability] = round(
(half_draw_model1[capability] /
categories[capability]) * 100, 2)
if capability not in half_draw_model2:
win_both_model2[capability] = 0.0
half_draw_model2[capability] = 0.0
else:
win_both_model2[capability] = round(
(win_both_model2[capability] /
categories[capability]) * 100, 2)
half_draw_model2[capability] = round(
(half_draw_model2[capability] /
categories[capability]) * 100, 2)
scores = {
'win_both_' + model1: win_both_model1,
'half_draw_' + model1: half_draw_model1,
'win_both_' + model2: win_both_model2,
'half_draw_' + model2: half_draw_model2
}
rows = list(scores.keys())
columns = list(scores[rows[0]].keys())
with open(fout, 'a+', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow([model1 + '_vs_' + model2] + columns)
for row in rows:
writer.writerow(
[row] +
[scores[row][column] for column in columns])
with open(fout, 'r') as f:
x = from_csv(f)
print(x)
opencompass/summarizers/subject.py → opencompass/summarizers/creationv01.py
View file @ 465308e4
# flake8: noqa: E501
import csv
import os
import os.path as osp
import re
from collections import defaultdict
from datetime import datetime
import mmengine
......@@ -14,7 +17,33 @@ except ImportError:
from opencompass.utils import dataset_abbr_from_cfg
class SubjectSummarizer:
def match_general_answer(s):
temp = s[0]
if temp in ['A', 'B', 'C', 'D']:
return temp
else:
return None
def match_GPT4_answer(s):
result = re.search(r'分数：(.)', s)
if result:
return int(result.group(1))
else:
return None
judge_map = {'smart': match_GPT4_answer, 'other': match_general_answer}
def call_function(name, arg):
if name in judge_map:
return judge_map[name](arg)
else:
print('Function not found in the map.')
class Creationv01Summarizer:
"""Do the subjectivity analyze based on evaluation results.
Args:
......@@ -22,12 +51,10 @@ class SubjectSummarizer:
It's expected to be filled out at runtime.
"""
def __init__(
self,
config: ConfigDict,
) -> None:
def __init__(self, config: ConfigDict, match_method='smart') -> None:
self.tasks = []
self.cfg = config
self.match_method = match_method
def summarize(self,
time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S')):
......@@ -49,32 +76,49 @@ class SubjectSummarizer:
output_dir = osp.join(osp.split(output_path)[0], f'{self.time_str}')
mmengine.mkdir_or_exist(output_dir)
results_folder = osp.join(work_dir, 'results')
fout = osp.join(output_dir, 'report.csv')
for subdir in os.listdir(results_folder):
subdir_path = os.path.join(results_folder, subdir)
if os.path.isdir(subdir_path):
model = subdir
for dataset in dataset_cfgs:
model1, model2 = dataset['eval_cfg']['evaluator'][
'base_model'], dataset['eval_cfg']['evaluator'][
'compare_model']
dataset_abbr = dataset_abbr_from_cfg(dataset)
filepath = os.path.join(subdir_path,
dataset_abbr + '.json')
result = mmengine.load(filepath)
rows = list(result.keys())
columns = list(result[rows[0]].keys())
fout = osp.join(output_dir,
model1 + '_vs_' + model2 + '.csv')
judged_answers = []
references = []
for k, v in result.items():
judged_answers.append(
call_function(self.match_method, v['prediction']))
references.append(v['gold'])
print(
'###############################Subjective Results on '
+ model1 + '_vs_' + model2 +
'###############################')
with open(fout, 'w', newline='') as csvfile:
f'Among {len(judged_answers)} judgements, successfully extracted {len(judged_answers)-judged_answers.count(None)} judgements.'
)
model_scores, categories = defaultdict(float), defaultdict(
float)
for prediction, reference in zip(judged_answers,
references):
categories[reference['capability']] += 1
if prediction is not None:
model_scores[reference['capability']] += prediction
for capability in categories:
if capability not in model_scores:
model_scores[capability] = 0.0
else:
model_scores[capability] = round(
model_scores[capability] /
categories[capability], 2)
scores = {model: model_scores}
rows = list(scores.keys())
columns = list(scores[rows[0]].keys())
with open(fout, 'a+', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow([model1 + '_vs_' + model2] + columns)
writer.writerow([''] + columns)
for row in rows:
writer.writerow(
[row] +
[result[row][column] for column in columns])
with open(fout, 'r') as f:
x = from_csv(f)
print(x)
[scores[row][column] for column in columns])
with open(fout, 'r') as f:
x = from_csv(f)
print(x)
opencompass/tasks/subjective_eval.py
View file @ 465308e4
......@@ -10,13 +10,11 @@ import mmengine
from mmengine.config import Config, ConfigDict
from mmengine.utils import mkdir_or_exist
from opencompass.openicl.icl_evaluator.lm_evaluator import LMEvaluator
from opencompass.registry import ICL_EVALUATORS, MODELS, TEXT_POSTPROCESSORS
from opencompass.tasks.base import BaseTask
from opencompass.utils import (build_dataset_from_cfg, dataset_abbr_from_cfg,
get_infer_output_path, get_logger,
task_abbr_from_cfg)
from opencompass.utils.types import get_type_from_cfg
class SubjectiveEvalTask(BaseTask):
......@@ -137,8 +135,7 @@ class SubjectiveEvalTask(BaseTask):
kwargs = pred_postprocessor or eval_cfg['pred_postprocessor']
proc = TEXT_POSTPROCESSORS.get(kwargs.pop('type'))
pred_strs = [proc(s, **kwargs) for s in pred_strs]
return pred_strs
return {'model_name': model_cfg['abbr'], 'model_preds': pred_strs}
def _score(self, model_cfg, dataset_cfg, eval_cfg, output_column):
test_set = build_dataset_from_cfg(dataset_cfg).test
......@@ -153,20 +150,15 @@ class SubjectiveEvalTask(BaseTask):
return sample
test_set = test_set.map(postprocess)
# Get out_path
out_path = get_infer_output_path(model_cfg, dataset_cfg,
osp.join(self.work_dir, 'results'))
model_preds = self._load_model_pred(model_cfg, dataset_cfg, eval_cfg)
if get_type_from_cfg(eval_cfg['evaluator']) == LMEvaluator:
if not self.judge_cfg:
raise ValueError('Using LMEvaluator in dataset, but '
'missing "eval.runner.task.judge_cfg" '
'as the judge configuration.')
eval_cfg['evaluator']['judge_cfg'] = self.judge_cfg
eval_cfg['evaluator']['dataset_cfg'] = dataset_cfg
eval_cfg['evaluator']['output_path'] = out_path
if not self.judge_cfg:
raise ValueError('missing "eval.runner.task.judge_cfg"')
eval_cfg['evaluator']['judge_cfg'] = self.judge_cfg
eval_cfg['evaluator']['dataset_cfg'] = dataset_cfg
eval_cfg['evaluator']['output_path'] = out_path
icl_evaluator = ICL_EVALUATORS.build(eval_cfg['evaluator'])
references = (test_set[output_column] if output_column else None)
result = icl_evaluator.score(predictions=model_preds,
......@@ -177,7 +169,8 @@ class SubjectiveEvalTask(BaseTask):
f'Task {task_abbr_from_cfg(self.cfg)}: {result["error"]}')
return
else:
self.logger.info(f'Task {task_abbr_from_cfg(self.cfg)}: {result}')
self.logger.info(
f'Task {task_abbr_from_cfg(self.cfg)}') #: {result}')
# Save result
mkdir_or_exist(osp.split(out_path)[0])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment