fix compass arena (#854)

configs/datasets/subjective/compassarena/compassarena_compare.py

@@ -88,19 +88,6 @@ math_prompt = """
 
 reason_prompt = math_prompt
 
-qa_prompt = """
-请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
-评分要求（重要性依次递减）:
-1. 好的回答必须首先具有事实正确性，即除了想象的内容外，所引用或阐述的各种信息都是真实正确的
-2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答，且前后连贯，逻辑没有问题
-3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误
-
-[用户问题]
-{question}
-""" + base_prompt
-
-
-
 creation_prompt = """
 请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
 评分要求（重要性依次递减）:
@@ -112,11 +99,9 @@ creation_prompt = """
 {question}
 """ + base_prompt
 
+sub_map = {"knowledge": knowledge_prompt, "language": language_prompt, "math_v2": math_prompt, "reason_v2": reason_prompt, "creationv2_zh": creation_prompt}
 
-subjective_all_sets = ["knowledge", "language", "math", "reason", "qa", "creationv2_zh"]
-prompt_all_sets = [knowledge_prompt, language_prompt, math_prompt, reason_prompt, qa_prompt, creation_prompt]
-
-for _name,_prompt in zip(subjective_all_sets, prompt_all_sets):
+for _name, _prompt in sub_map.items():
     subjective_infer_cfg = dict(
             prompt_template=dict(
                 type=PromptTemplate,

configs/eval_subjective_compassarena.py

@@ -3,7 +3,6 @@ from opencompass.models import HuggingFaceCausalLM
 from mmengine.config import read_base
 with read_base():
     from .models.chatglm.hf_chatglm3_6b_32k import models as chatglm3_6b_32k_model
-    from .models.yi.hf_yi_6b_chat import models as yi_6b_chat_model
     from .datasets.subjective.compassarena.compassarena_compare import subjective_datasets
 
 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
@@ -19,7 +18,7 @@ from opencompass.summarizers import CompassArenaSummarizer
 
 infer = dict(
     #partitioner=dict(type=NaivePartitioner),
-    partitioner=dict(type=SizePartitioner, max_task_size=10000),
+    partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000),
     runner=dict(
         type=SlurmSequentialRunner,
         partition='llm_dev2',
@@ -47,12 +46,12 @@ gpt4 = dict(
         retry=20,
         temperature = 1
 )
-models = [*chatglm3_6b_32k_model, *yi_6b_chat_model]
+models = [*chatglm3_6b_32k_model]
 datasets = [*subjective_datasets]
 
 
 
-work_dir = 'outputs/compass_arena/'
+work_dir = 'outputs/compass_arena_debug/'
 
 # -------------Inferen Stage ----------------------------------------
 
@@ -68,6 +67,7 @@ judge_model = dict(
         retry=20,
         temperature = 0
 )
+
 ## ------------- Evaluation Configuration
 eval = dict(
     partitioner=dict(
@@ -76,7 +76,7 @@ eval = dict(
         max_task_size=10000,
         mode='m2n',
         base_models = [gpt4],
-        compare_models = [*chatglm3_6b_32k_model, *yi_6b_chat_model, ]
+        compare_models = [*chatglm3_6b_32k_model]
     ),
     runner=dict(
         type=SlurmSequentialRunner,
@@ -91,5 +91,6 @@ eval = dict(
 
 
 summarizer = dict(
-    type=CompassArenaSummarizer
+    type=CompassArenaSummarizer,
+    summary_type='half_add'
 )
\ No newline at end of file

opencompass/summarizers/subjective/compass_arena.py

@@ -35,7 +35,7 @@ def check_position_bias(judged_answers, references, banned_choice=['C']):
     position_bias_flag = 0
     position_bias_dict = {}
     for judge, ref in zip(judged_answers, references):
-        question = ref['others']['question']
+        question = ref['question']
         question_hash = hash(question)
         if question_hash not in position_bias_dict:
             position_bias_dict[question_hash] = {
@@ -58,7 +58,11 @@ class CompassArenaSummarizer:
             It's expected to be filled out at runtime.
     """
 
-    def __init__(self, config: ConfigDict, judge_type='general') -> None:
+    def __init__(self,
+                 config: ConfigDict,
+                 judge_type='general',
+                 check_pos_bias=True,
+                 summary_type='single') -> None:
         self.tasks = []
         self.cfg = config
         self.base_models = self.cfg['eval']['partitioner']['base_models']
@@ -70,10 +74,13 @@ class CompassArenaSummarizer:
             'general': post_process_compass_arena,
         }
         self.judge_function = self.judge_map[self.judge_type]
+        self.check_pos_bias = check_pos_bias
+        self.summary_type = summary_type
 
-    def summarize(self,
-                  time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S'),
-                  check_pos_bias=True):
+    def summarize(
+            self,
+            time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S'),
+    ):
         """Summarize the subjectivity analysis based on evaluation results.
 
         Args:
@@ -88,25 +95,25 @@ class CompassArenaSummarizer:
             product(self.base_models, self.compare_models))
         unique_combinations = remove_duplicate_pairs(
             [combo for combo in model_combinations if combo[0] != combo[1]])
+        judge_model = self.judge_abbr
         fout_list = []
-        for model_pair in unique_combinations:
-            model1, model2, judge_model = model_pair[0]['abbr'], model_pair[1][
-                'abbr'], self.judge_abbr
-            subdir = model1 + '_' + model2 + '_judged-by--' + self.judge_abbr
-            subdir_path = os.path.join(results_folder, subdir)
-            if os.path.isdir(subdir_path):
-                for dataset in dataset_cfgs:
-                    dataset_abbr = dataset_abbr_from_cfg(dataset)
-                    fout = osp.join(
-                        output_dir, 'judged-by--' + judge_model + '-' +
-                        dataset_abbr + '-report.csv')
-                    fout_list.append(fout)
+        for dataset in dataset_cfgs:
+            dataset_abbr = dataset_abbr_from_cfg(dataset)
+            fout = osp.join(
+                output_dir, 'judged-by--' + judge_model + '-' + dataset_abbr +
+                '-report.csv')
+            fout_list.append(fout)
+            for model_pair in unique_combinations:
+                model1, model2, = model_pair[0]['abbr'], model_pair[1]['abbr'],
+                subdir = model1 + '_' + model2 + '_judged-by--' + judge_model
+                subdir_path = os.path.join(results_folder, subdir)
+                if os.path.isdir(subdir_path):
                     judged_answers, references = get_judgeanswer_and_reference(
                         dataset,
                         subdir_path,
                         self.judge_function,
                     )
-                    if check_pos_bias:
+                    if self.check_pos_bias:
                         bias_num = check_position_bias(judged_answers,
                                                        references)
                     else:
@@ -117,24 +124,47 @@ class CompassArenaSummarizer:
                         'answer2']
                     for prediction, reference in zip(judged_answers,
                                                      references):
-                        if dataset_abbr == 'qa':
-                            reference['capability'] = 'QA'
-                        categories['total'] += 1
-                        categories[reference['capability']] += 1
-                        if prediction == 'A':
-                            if reference['answer1'] == model1:
-                                win_model1[reference['capability']] += 1
-                                win_model1['total'] += 1
-                            else:
-                                win_model2[reference['capability']] += 1
-                                win_model2['total'] += 1
-                        elif prediction == 'B':
-                            if reference['answer1'] == model1:
-                                win_model2[reference['capability']] += 1
-                                win_model2['total'] += 1
-                            else:
-                                win_model1[reference['capability']] += 1
-                                win_model1['total'] += 1
+                        if self.summary_type == 'single':
+                            if prediction == 'A':
+                                categories['total'] += 1
+                                categories[reference['capability']] += 1
+                                if reference['answer1'] == model1:
+                                    win_model1[reference['capability']] += 1
+                                    win_model1['total'] += 1
+                                else:
+                                    win_model2[reference['capability']] += 1
+                                    win_model2['total'] += 1
+                            elif prediction == 'B':
+                                categories['total'] += 1
+                                categories[reference['capability']] += 1
+                                if reference['answer1'] == model1:
+                                    win_model2[reference['capability']] += 1
+                                    win_model2['total'] += 1
+                                else:
+                                    win_model1[reference['capability']] += 1
+                                    win_model1['total'] += 1
+                        elif self.summary_type == 'half_add':
+                            categories['total'] += 1
+                            categories[reference['capability']] += 1
+                            if prediction == 'A':
+                                if reference['answer1'] == model1:
+                                    win_model1[reference['capability']] += 1
+                                    win_model1['total'] += 1
+                                else:
+                                    win_model2[reference['capability']] += 1
+                                    win_model2['total'] += 1
+                            elif prediction == 'B':
+                                if reference['answer1'] == model1:
+                                    win_model2[reference['capability']] += 1
+                                    win_model2['total'] += 1
+                                else:
+                                    win_model1[reference['capability']] += 1
+                                    win_model1['total'] += 1
+                            elif prediction == 'C':
+                                win_model1[reference['capability']] += 0.5
+                                win_model1['total'] += 0.5
+                                win_model2[reference['capability']] += 0.5
+                                win_model2['total'] += 0.5
                     for capability in categories:
                         if capability not in win_model1:
                             win_model1[capability] = 0.0
@@ -166,8 +196,8 @@ class CompassArenaSummarizer:
                             writer.writerow(
                                 [row] +
                                 [scores[row][column] for column in columns])
-            else:
-                print(subdir_path + ' is not exist! please check!')
+                else:
+                    print(subdir_path + ' is not exist! please check!')
         for fout in fout_list:
             with open(fout, 'r') as f:
                 x = from_csv(f)

opencompass/summarizers/subjective/utils.py

@@ -64,9 +64,10 @@ def get_judgeanswer_and_reference(dataset, subdir_path, post_process):
         if processed_judge is not None:
             judged_answers.append(processed_judge)
             references.append(v['gold'])
-    print(
-        f'Among {len(result)} judgements, successfully extracted {len(judged_answers)} judgements.'
-    )
+    if len(judged_answers) != len(result):
+        print(
+            f'Among {len(result)} judgements, successfully extracted {len(judged_answers)} judgements, please check!'
+        )
     if len(judged_answers) == 0:
         print('*' * 100)
         print(