fix arenahard summarizer (#1154)

Co-authored-by: Leymore <zfz-960727@163.com>

opencompass/summarizers/subjective/arenahard.py

@@ -50,8 +50,7 @@ def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
     tie_idx = (df['winner'] == 'tie') | (df['winner'] == 'tie (bothbad)')
     tie_idx[len(tie_idx)//2:] = False
     Y[tie_idx] = 1.0
-
-    lr = LogisticRegression(fit_intercept=False, penalty=None, tol=1e-8)
+    lr = LogisticRegression(fit_intercept=False, penalty=None, tol=1e-8) # May need to set a small value when not use GPT4 as judge model
     lr.fit(X,Y)
 
     elo_scores = SCALE * lr.coef_[0] + INIT_RATING
@@ -134,10 +133,8 @@ def get_win_rate_column(df, column, baseline='gpt4-0314'):
     return win_rate_table[baseline].fillna(0.5).apply(lambda x: round(x * 100, 2))
 
 
-def get_battles_from_judgment(dataset, subdir_path, post_process, first_game_only=False, WEIGHT=3):
+def get_battles_from_judgment(dataset, subdir_path, post_process, WEIGHT=3):
     arena_hard_battles = pd.DataFrame()
-    print('Turning judgment results into battles...')
-
     dataset_abbr = dataset_abbr_from_cfg(dataset)
     filename = osp.join(subdir_path, dataset_abbr + '.json')
     partial_filename = osp.join(subdir_path, dataset_abbr + '_0.json')
@@ -197,7 +194,6 @@ def get_battles_from_judgment(dataset, subdir_path, post_process, first_game_onl
         if weight:
             arena_hard_battles = pd.concat([arena_hard_battles, pd.DataFrame([output] * weight)])
 
-    arena_hard_battles.to_json(os.path.join(subdir_path,'arena_hard_battles.jsonl'), lines=True, orient='records')
     return arena_hard_battles
 
 class ArenaHardSummarizer:
@@ -240,6 +236,8 @@ class ArenaHardSummarizer:
             judge_model = model_abbr_from_cfg(judge_model_cfg)
             for dataset in self.cfg['datasets']:
                 dataset_abbr = dataset_abbr_from_cfg(dataset)
+                battles = pd.DataFrame()
+                print('Turning judgment results into battles...')
                 for model_pair in unique_combinations:
                     model1 = model_pair[0]['abbr']
                     model2 = model_pair[1]['abbr']
@@ -252,13 +250,15 @@ class ArenaHardSummarizer:
                         print(subdir_path + ' is not exist! please check!')
                         continue
 
-                    battles = get_battles_from_judgment(dataset, subdir_path, self.judge_function)
+                    new_battle = get_battles_from_judgment(dataset, subdir_path, self.judge_function)
+                    battles = pd.concat([battles, new_battle], ignore_index=True)
+                battles.to_json(os.path.join(output_dir,'arena_hard_battles_judged-by--'+ judge_model+'.jsonl'), lines=True, orient='records')
 
                 bootstrap_online_elo = compute_mle_elo(battles)
 
                 np.random.seed(42)
                 bootstrap_elo_lu = get_bootstrap_result(battles, compute_mle_elo, 100)
-                    bootstrap_elo_lu.to_json(os.path.join(subdir_path,'bootstrapping_results.jsonl'), lines=True, orient='records')
+                bootstrap_elo_lu.to_json(os.path.join(output_dir,'bootstrapping_results'+ judge_model+'.jsonl'), lines=True, orient='records')
 
                 stats = pd.DataFrame()
                 stats['results'] = None
@@ -287,14 +287,12 @@ class ArenaHardSummarizer:
                 stats['lower'] = get_win_rate_column(stats, 'lower', 'gpt4-0314').tolist()
                 stats['upper'] = get_win_rate_column(stats, 'upper', 'gpt4-0314').tolist()
                 decimal = 1
-
-
                 stats.sort_values(by='score', ascending=False, inplace=True)
                 for _, row in stats.iterrows():
                     interval = str((round(row['lower'] - row['score'], decimal), round(row['upper'] - row['score'], decimal)))
                     print(f"{row['model'] : <30} | score: {round(row['score'], decimal) : ^5} | 95% CI: {interval : ^12} | average #tokens: {int(row['avg_tokens'])}")
-
-                    stats.to_json(os.path.join(output_dir,'arena_hard_leaderboard.json'), orient='records', indent=4)
+                stats.to_json(os.path.join(output_dir,'arena_hard_leaderboard_judged-by--'+judge_model+'.json'), orient='records', indent=4)
+                stats.to_csv(os.path.join(output_dir,'arena_hard_leaderboard_judged-by--'+judge_model+'.csv'))
 
     def summarize(
             self,