[Feature] Add circular eval (#610)

* refactor default, add circular summarizer

* add circular

* update impl

* update doc

* minor update

* no more to be added

configs/eval_circular.py

+from mmengine.config import read_base
+from opencompass.datasets.circular import (CircularCEvalDataset, CircularMMLUDataset, CircularCMMLUDataset, CircularCSQADataset,
+                                           CircularARCDataset, CircularHSWAGDataset, CircularOBQADataset, CircularRaceDataset, CircularEvaluator)
+from opencompass.summarizers import CircularSummarizer
+
+with read_base():
+    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+    from .datasets.mmlu.mmlu_gen_a484b3 import mmlu_datasets
+    from .datasets.cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
+    from .datasets.hellaswag.hellaswag_gen_6faab5 import hellaswag_datasets
+    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets
+    from .datasets.commonsenseqa.commonsenseqa_gen_1da2d0 import commonsenseqa_datasets
+    from .datasets.obqa.obqa_gen_9069e4 import obqa_datasets
+    from .datasets.race.race_gen_69ee4f import race_datasets
+
+    from .models.hf_internlm.hf_internlm_chat_7b import models as hf_internlm_chat_7b_model
+    from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b_model
+    from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat_model
+    from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat_model
+
+    from .summarizers.groups.mmlu import mmlu_summary_groups
+    from .summarizers.groups.cmmlu import cmmlu_summary_groups
+    from .summarizers.groups.ceval import ceval_summary_groups
+
+for ds, t in [
+    (ceval_datasets, CircularCEvalDataset),
+    (mmlu_datasets, CircularMMLUDataset),
+    (cmmlu_datasets, CircularCMMLUDataset),
+    (hellaswag_datasets, CircularHSWAGDataset),
+    (ARC_e_datasets, CircularARCDataset),
+    (ARC_c_datasets, CircularARCDataset),
+    (commonsenseqa_datasets, CircularCSQADataset),
+    (obqa_datasets, CircularOBQADataset),
+    (race_datasets, CircularRaceDataset),
+]:
+    for d in ds:
+        d['type'] = t
+        d['abbr'] = d['abbr'] + '-circular-4'
+        d['eval_cfg']['evaluator'] = {'type': CircularEvaluator, 'circular_pattern': 'circular'}
+        d['circular_patterns'] = 'circular'
+
+
+datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
+models = sum([v for k, v in locals().items() if k.endswith("_model")], [])
+
+# config summarizer
+other_summary_groups = [
+    {'name': 'average',
+     'subsets': ['ceval', 'mmlu', 'cmmlu', 'hellaswag', 'ARC-e', 'ARC-c', 'commonsense_qa', 'openbookqa_fact', 'race-middle', 'race-high']},
+]
+origin_summary_groups = sum([v for k, v in locals().items() if k.endswith("_summary_groups")], [])
+new_summary_groups = []
+for item in origin_summary_groups:
+    new_summary_groups.append(
+        {
+            'name': item['name'] + '-circular-4',
+            'subsets': [i + '-circular-4' for i in item['subsets']],
+        }
+    )
+summarizer = dict(
+    type=CircularSummarizer,
+    metric_types=['acc_origin', 'perf_circular'],
+    dataset_abbrs = [
+        'average-circular-4',
+        'ceval-circular-4',
+        'mmlu-circular-4',
+        'cmmlu-circular-4',
+        'hellaswag-circular-4',
+        'ARC-e-circular-4',
+        'ARC-c-circular-4',
+        'commonsense_qa-circular-4',
+        'openbookqa_fact-circular-4',
+        'race-middle-circular-4',
+        'race-high-circular-4',
+        'ceval-humanities-circular-4',
+        'ceval-stem-circular-4',
+        'ceval-social-science-circular-4',
+        'ceval-other-circular-4',
+        'mmlu-humanities-circular-4',
+        'mmlu-stem-circular-4',
+        'mmlu-social-science-circular-4',
+        'mmlu-other-circular-4',
+        'cmmlu-humanities-circular-4',
+        'cmmlu-stem-circular-4',
+        'cmmlu-social-science-circular-4',
+        'cmmlu-other-circular-4',
+        'cmmlu-china-specific-circular-4',
+    ],
+    summary_groups=new_summary_groups,
+)

docs/en/advanced_guides/circular_eval.md

+# CircularEval
+
+## Background
+
+For multiple-choice questions, when a Language Model (LLM) provides the correct option, it does not necessarily imply a true understanding and reasoning of the question. It could be a guess. To differentiate these scenarios and reduce LLM bias towards options, CircularEval (CircularEval) can be utilized. A multiple-choice question is augmented by shuffling its options, and if the LLM correctly answers all variations of the augmented question, it is considered correct under CircularEval.
+
+## Adding Your Own CircularEval Dataset
+
+Generally, to evaluate a dataset using CircularEval, both its loading and evaluation methods need to be rewritten. Modifications are required in both the OpenCompass main library and configuration files. We will use C-Eval as an example for explanation.
+
+OpenCompass main library:
+
+```python
+from opencompass.datasets.ceval import CEvalDataset
+from opencompass.datasets.circular import CircularDatasetMeta
+
+class CircularCEvalDataset(CEvalDataset, metaclass=CircularDatasetMeta):
+    # The overloaded dataset class
+    dataset_class = CEvalDataset
+
+    # Splits of the DatasetDict that need CircularEval. For CEvalDataset, which loads [dev, val, test], we only need 'val' and 'test' for CircularEval, not 'dev'
+    default_circular_splits = ['val', 'test']
+
+    # List of keys to be shuffled
+    default_option_keys = ['A', 'B', 'C', 'D']
+
+    # If the content of 'answer_key' is one of ['A', 'B', 'C', 'D'], representing the correct answer. This field indicates how to update the correct answer after shuffling options. Choose either this or default_answer_key_switch_method
+    default_answer_key = 'answer'
+
+    # If 'answer_key' content is not one of ['A', 'B', 'C', 'D'], a function can be used to specify the correct answer after shuffling options. Choose either this or default_answer_key
+    # def default_answer_key_switch_method(item, circular_pattern):
+    #     # 'item' is the original data item
+    #     # 'circular_pattern' is a tuple indicating the order after shuffling options, e.g., ('D', 'A', 'B', 'C') means the original option A is now D, and so on
+    #     item['answer'] = circular_pattern['ABCD'.index(item['answer'])]
+    #     return item
+```
+
+`CircularCEvalDataset` accepts the `circular_pattern` parameter with two values:
+
+- `circular`: Indicates a single cycle. It is the default value. ABCD is expanded to ABCD, BCDA, CDAB, DABC, a total of 4 variations.
+- `all_possible`: Indicates all permutations. ABCD is expanded to ABCD, ABDC, ACBD, ACDB, ADBC, ADCB, BACD, ..., a total of 24 variations.
+
+Additionally, we provide a `CircularEvaluator` to replace `AccEvaluator`. This Evaluator also accepts `circular_pattern`, and it should be consistent with the above. It produces the following metrics:
+
+- `acc_{origin|circular|all_possible}`: Treating each question with shuffled options as separate, calculating accuracy.
+- `perf_{origin|circular|all_possible}`: Following Circular logic, a question is considered correct only if all its variations with shuffled options are answered correctly, calculating accuracy.
+- `more_{num}_{origin|circular|all_possible}`: According to Circular logic, a question is deemed correct if the number of its variations answered correctly is greater than or equal to num, calculating accuracy.
+
+OpenCompass configuration file:
+
+```python
+from mmengine.config import read_base
+from opencompass.datasets.circular import CircularCEvalDataset
+
+with read_base():
+    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+
+for d in ceval_datasets:
+    # Overloading the load method
+    d['type'] = CircularCEvalDataset
+    # Renaming for differentiation from non-circular evaluation versions
+    d['abbr'] = d['abbr'] + '-circular-4'
+    # Overloading the evaluation method
+    d['eval_cfg']['evaluator'] = {'type': CircularEvaluator}
+
+# The dataset after the above operations looks like this:
+# dict(
+#     type=CircularCEvalDataset,
+#     path='./data/ceval/formal_ceval',  # Unchanged
+#     name='computer_network',  # Unchanged
+#     abbr='ceval-computer_network-circular-4',
+#     reader_cfg=dict(...),  # Unchanged
+#     infer_cfg=dict(...),  # Unchanged
+#     eval_cfg=dict(evaluator=dict(type=CircularEvaluator), ...),
+# )
+```
+
+Additionally, for better presentation of results in CircularEval, consider using the following summarizer:
+
+```python
+
+
+from mmengine.config import read_base
+from opencompass.summarizers import CircularSummarizer
+
+with read_base():
+    from ...summarizers.groups.ceval.ceval_summary_groups
+
+new_summary_groups = []
+for item in ceval_summary_groups:
+    new_summary_groups.append(
+        {
+            'name': item['name'] + '-circular-4',
+            'subsets': [i + '-circular-4' for i in item['subsets']],
+        }
+    )
+
+summarizer = dict(
+    type=CircularSummarizer,
+    # Select specific metrics to view
+    metric_types=['acc_origin', 'perf_circular'],
+    dataset_abbrs = [
+        'ceval-circular-4',
+        'ceval-humanities-circular-4',
+        'ceval-stem-circular-4',
+        'ceval-social-science-circular-4',
+        'ceval-other-circular-4',
+    ],
+    summary_groups=new_summary_groups,
+)
+```
+
+For more complex evaluation examples, refer to this sample code: https://github.com/open-compass/opencompass/tree/main/configs/eval_circular.py

docs/en/index.rst

@@ -67,6 +67,7 @@ We always welcome *PRs* and *Issues* for the betterment of OpenCompass.
    advanced_guides/prompt_attack.md
    advanced_guides/longeval.md
    advanced_guides/subjective_evaluation.md
+   advanced_guides/circular_eval.md
 
 .. _Tools:
 .. toctree::

docs/zh_cn/advanced_guides/circular_eval.md

+# 循环评测
+
+## 背景
+
+对于选择题而言，当 LLM 给出正确的选项，并不一定代表着它能真正地理解题意并经过推理得出答案，它也有可能是蒙对的。为了将这两种情形区分开，同时也为了降低 LLM 对选项的偏见，我们可以尝试使用循环评测 (CircularEval)。我们会将一道选择题按照打乱选项的方式进行增广，若 LLM 可以在增广后的每道题上均得到正确的答案，那么我们认为在循环评测的意义下，这道题被做对了。
+
+## 新增自己的循环评测数据集
+
+一般来说，为了将一个数据集使用循环评测的方式进行评测，它的加载方式和评测方式是需要被重写的，OpenCompass 主库和配置文件均需要进行修改。后续我们以 C-Eval 为例进行讲解。
+
+OpenCompass 主库：
+
+```python
+from opencompass.datasets.ceval import CEvalDataset
+from opencompass.datasets.circular import CircularDatasetMeta
+
+class CircularCEvalDataset(CEvalDataset, metaclass=CircularDatasetMeta):
+    # 被重载的数据集类
+    dataset_class = CEvalDataset
+
+    # 若原 load 方法得到一 DatasetDict，其哪些 split 需要被循环评测。CEvalDataset load 得到 [dev, val, test]，我们只需要对 val 和 test 进行循环评测，dev 不需要
+    default_circular_splits = ['val', 'test']
+
+    # 需要被打乱的 key 列表
+    default_option_keys = ['A', 'B', 'C', 'D']
+
+    # 若 answer_key 的内容属于是 ['A', 'B', 'C', 'D'] 之一，并表示正确答案。该字段表示打乱选项后，需要如何更新正确答案。与 default_answer_key_switch_method 二选一
+    default_answer_key = 'answer'
+
+    # 如果 answer_key 的内容不属于 ['A', 'B', 'C', 'D'] 之一，那么可以使用函数的方式来指定打乱选项后的正确答案。与 default_answer_key 二选一
+    # def default_answer_key_switch_method(item, circular_pattern):
+    #     # item 是原本的数据项
+    #     # circular_pattern 是一个 tuple，表示打乱选项后的顺序，例如 ('D', 'A', 'B', 'C') 表示原来的 A 选项变成了 D，原来的 B 选项变成了 A，以此类推
+    #     item['answer'] = circular_pattern['ABCD'.index(item['answer'])]
+    #     return item
+```
+
+`CircularCEvalDataset` 会接受 `circular_pattern` 参数，它有两个取值:
+
+- `circular`: 表示单项循环。默认为该值。ABCD 会被扩充为 ABCD, BCDA, CDAB, DABC, 共 4 种
+- `all_possible`: 表示全排列。ABCD 会被扩充为 ABCD, ABDC, ACBD, ACDB, ADBC, ADCB, BACD, ..., 共 24 种
+
+另外我们提供了一个 `CircularEvaluator` 用于替换 `AccEvaluator`，该 Evaluator 同样接受 `circular_pattern`，该参数应与上述保持一致。它会产出以下指标：
+
+- `acc_{origin|circular|all_possible}`: 将打乱后选项顺序后的题目视作多道单独的题目，计算准确率
+- `perf_{origin|circular|all_possible}`: 按照 circular 的逻辑，若选项打乱后的题目都回答正确，才会视为这道题正确，计算准确率
+- `more_{num}_{origin|circular|all_possible}`: 按照 circular 的逻辑，若选项打乱后的题目回答正确的数量大于等于 num，就会视为这道题正确，计算准确率
+
+OpenCompass 配置文件：
+
+```python
+from mmengine.config import read_base
+from opencompass.datasets.circular import CircularCEvalDataset
+
+with read_base():
+    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+
+for d in ceval_datasets:
+    # 重载 load 方法
+    d['type'] = CircularCEvalDataset
+    # 为了与非循环评测版本做区分而进行改名
+    d['abbr'] = d['abbr'] + '-circular-4'
+    # 重载评测方法
+    d['eval_cfg']['evaluator'] = {'type': CircularEvaluator}
+
+# 上述操作后的 dataset 形如下：
+# dict(
+#     type=CircularCEvalDataset,
+#     path='./data/ceval/formal_ceval',  # 未改变
+#     name='computer_network',  # 未改变
+#     abbr='ceval-computer_network-circular-4',
+#     reader_cfg=dict(...),  # 未改变
+#     infer_cfg=dict(...),  # 未改变
+#     eval_cfg=dict(evaluator=dict(type=CircularEvaluator), ...),
+# )
+```
+
+另外评测时为了针对循环评测有更良好的结果呈现，建议考虑使用以下 summarizer
+
+```python
+from mmengine.config import read_base
+from opencompass.summarizers import CircularSummarizer
+
+with read_base():
+    from ...summarizers.groups.ceval import ceval_summary_groups
+
+new_summary_groups = []
+for item in ceval_summary_groups:
+    new_summary_groups.append(
+        {
+            'name': item['name'] + '-circular-4',
+            'subsets': [i + '-circular-4' for i in item['subsets']],
+        }
+    )
+
+summarizer = dict(
+    type=CircularSummarizer,
+    # 选择具体看哪些指标
+    metric_types=['acc_origin', 'perf_circular'],
+    dataset_abbrs = [
+        'ceval-circular-4',
+        'ceval-humanities-circular-4',
+        'ceval-stem-circular-4',
+        'ceval-social-science-circular-4',
+        'ceval-other-circular-4',
+    ],
+    summary_groups=new_summary_groups,
+)
+```
+
+更多复杂的评测案例可以参考这个样例代码: https://github.com/open-compass/opencompass/tree/main/configs/eval_circular.py

docs/zh_cn/index.rst

@@ -67,6 +67,7 @@ OpenCompass 上手路线
    advanced_guides/prompt_attack.md
    advanced_guides/longeval.md
    advanced_guides/subjective_evaluation.md
+   advanced_guides/circular_eval.md
 
 .. _工具:
 .. toctree::

opencompass/datasets/__init__.py

@@ -13,6 +13,7 @@ from .cb import *  # noqa: F401, F403
 from .ceval import *  # noqa: F401, F403
 from .chid import *  # noqa: F401, F403
 from .cibench import *  # noqa: F401, F403
+from .circular import *  # noqa: F401, F403
 from .civilcomments import *  # noqa: F401, F403
 from .clozeTest_maxmin import *  # noqa: F401, F403
 from .cluewsc import *  # noqa: F401, F403

opencompass/datasets/arc.py

@@ -14,32 +14,19 @@ class ARCDataset(BaseDataset):
     def load(path: str):
         with open(path, 'r', errors='ignore') as in_f:
             rows = []
-            for i, line in enumerate(in_f):
-                sample = json.loads(line.strip())
-                answerKey = sample['answerKey']
-                sample = sample['question']
-                question = sample['stem']
-                choices = sample['choices']
-                if len(choices) != 4:
+            for line in in_f:
+                item = json.loads(line.strip())
+                question = item['question']
+                if len(question['choices']) != 4:
                     continue
-                textA = choices[0]['text']
-                textB = choices[1]['text']
-                textC = choices[2]['text']
-                textD = choices[3]['text']
+                labels = [c['label'] for c in question['choices']]
+                answerKey = 'ABCD'[labels.index(item['answerKey'])]
                 rows.append({
-                    'question': question,
+                    'question': question['stem'],
                     'answerKey': answerKey,
-                    'textA': textA,
-                    'textB': textB,
-                    'textC': textC,
-                    'textD': textD
+                    'textA': question['choices'][0]['text'],
+                    'textB': question['choices'][1]['text'],
+                    'textC': question['choices'][2]['text'],
+                    'textD': question['choices'][3]['text'],
                 })
-            dataset = Dataset.from_dict({
-                'question': [row['question'] for row in rows],
-                'answerKey': [row['answerKey'] for row in rows],
-                'textA': [row['textA'] for row in rows],
-                'textB': [row['textB'] for row in rows],
-                'textC': [row['textC'] for row in rows],
-                'textD': [row['textD'] for row in rows]
-            })
-            return dataset
+            return Dataset.from_list(rows)

opencompass/datasets/circular.py

+import copy
+import itertools
+from typing import Callable, List, Optional, Union
+
+from datasets import Dataset, DatasetDict
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+
+from .arc import ARCDataset
+from .ceval import CEvalDataset
+from .cmmlu import CMMLUDataset
+from .commonsenseqa import commonsenseqaDataset
+from .hellaswag import hellaswagDataset_V2
+from .mmlu import MMLUDataset
+from .obqa import OBQADataset
+from .race import RaceDataset
+from .xiezhi import XiezhiDataset
+
+
+def get_origin_patterns(option_keys):
+    return [tuple(option_keys)]
+
+
+def get_circular_patterns(option_keys):
+    double_option_keys = option_keys + option_keys
+    circular_patterns = [
+        tuple(double_option_keys[i:i + len(option_keys)])
+        for i in range(len(option_keys))
+    ]
+    return circular_patterns
+
+
+def get_all_possible_patterns(option_keys):
+    circular_patterns = list(itertools.permutations(option_keys))
+    return circular_patterns
+
+
+class CircularDatasetMeta(type):
+    """This Meta Class is designed to transform a class that reads datasets
+    into one that supports reading datasets required for CircularEval. It
+    overloads an existing load method for the original class.
+
+    The Meta Class should possess the following attributes:
+
+    - `dataset_class` (class): The class for reading datasets, such as
+        `CEvalDataset`.
+    - `default_circular_splits` (list, optional): The default splits of the
+        dataset that need to undergo CircularEval, like ['val', 'test']. If a
+        `Dataset` is loaded originally, this field will be ignored.
+    - `default_option_keys` (list): The keys for options in the dataset, such
+        as ['A', 'B', 'C', 'D'].
+    - `default_answer_key` (str, optional): The key for answers in the dataset,
+        like 'answer'. This is an alternative to
+        `default_answer_key_switch_method`.
+    - `default_answer_key_switch_method` (function, optional): The method to
+        transform the key for answers in the dataset. This is an alternative to
+        `default_answer_key`.
+    """
+
+    @staticmethod
+    def make_circular_items(
+        origin_item,
+        circular_patterns,
+        option_keys,
+        answer_key,
+        answer_key_switch_method,
+        qid,
+    ):
+        items = []
+        for circular_pattern in circular_patterns:
+            item = copy.deepcopy(origin_item)
+            for i in range(len(option_keys)):
+                item[circular_pattern[i]] = origin_item[option_keys[i]]
+            if answer_key_switch_method is None:
+                if origin_item[answer_key] in option_keys:
+                    item[answer_key] = circular_pattern[option_keys.index(
+                        origin_item[answer_key])]
+                else:
+                    pass
+            else:
+                item = answer_key_switch_method(item, circular_pattern)
+            item['qid'] = qid
+            item['circular_pattern'] = tuple(circular_pattern)
+            items.append(item)
+        return items
+
+    @staticmethod
+    def make_circular_dataset(dataset, circular_patterns, option_keys,
+                              answer_key, answer_key_switch_method):
+        circulated_items = []
+        for i, item in enumerate(dataset):
+            item = CircularDatasetMeta.make_circular_items(
+                item,
+                circular_patterns,
+                option_keys,
+                answer_key,
+                answer_key_switch_method,
+                i,
+            )
+            circulated_items.extend(item)
+        return Dataset.from_list(circulated_items)
+
+    def make_circular(
+        dataset: Union[Dataset, DatasetDict],
+        circular_splits: Optional[List[str]] = ['test'],
+        circular_patterns: str = 'circular',
+        option_keys: List[str] = ['A', 'B', 'C', 'D'],
+        answer_key: Optional[str] = 'answer',
+        answer_key_switch_method: Optional[Callable] = None,
+    ):
+        """Transform the dataset into one that is compatible with CircularEval.
+        In CircularEval, the original multiple-choice questions with options
+        ABCD are augmented by shuffling the order of options, such as BCDA,
+        CDAB, DABC, etc. A model is considered correct only if it answers all
+        augmented questions correctly. This method effectively prevents models
+        from memorizing answers.
+
+        Args:
+        datasets: The dataset to be augmented.
+        circular_splits: List of splits to make circular. This is only
+            effective when the dataset is a DatasetDict.
+        circular_patterns: Method for circular processing, can be 'circular'
+            for single cycle or 'all_possible' for all permutations, default
+            is 'circular'.
+        option_keys: List of keys for options, default to ['A', 'B', 'C', 'D'].
+        answer_key: Key for the answer, default to 'answer'. When specified,
+            ensure that the content of answer_key is among the option_keys.
+            It is an alternative to specifying answer_key_switch_method.
+        answer_key_switch_method: Function to modify the answer_key. It is an
+            alternative to specifying answer_key.
+        """
+
+        if isinstance(circular_patterns, str):
+            if circular_patterns == 'circular':
+                circular_patterns = get_circular_patterns(option_keys)
+            elif circular_patterns == 'all_possible':
+                circular_patterns = get_all_possible_patterns(option_keys)
+            else:
+                raise ValueError(
+                    f'Unknown circular_patterns: {circular_patterns}')
+        else:
+            assert isinstance(circular_patterns, list)
+            assert all([isinstance(i, list) for i in circular_patterns])
+            # TODO: other necessary sanity checks
+            raise NotImplementedError(
+                'circular_patterns int list of list has not been tested yet')
+
+        if answer_key is None and answer_key_switch_method is None:
+            raise ValueError(
+                'answer_key and answer_key_switch_method cannot be both None')
+        if answer_key is not None and answer_key_switch_method is not None:
+            raise ValueError(
+                'either answer_key or answer_key_switch_method should be None')
+
+        if isinstance(dataset, Dataset):
+            dataset = CircularDatasetMeta.make_circular_dataset(
+                dataset,
+                circular_patterns,
+                option_keys,
+                answer_key,
+                answer_key_switch_method,
+            )
+        else:
+            assert isinstance(dataset, DatasetDict)
+            dataset_dict = {}
+            for split in dataset:
+                if circular_splits is not None and split in circular_splits:
+                    dataset_dict[
+                        split] = CircularDatasetMeta.make_circular_dataset(
+                            dataset[split],
+                            circular_patterns,
+                            option_keys,
+                            answer_key,
+                            answer_key_switch_method,
+                        )
+                else:
+                    dataset_dict[split] = dataset[split]
+            dataset = DatasetDict(dataset_dict)
+        return dataset
+
+    def __new__(cls, name, bases, dct):
+        new_cls = super().__new__(cls, name, bases, dct)
+
+        def load(cls, circular_patterns='circular', *args, **kwargs):
+            circular_splits = getattr(cls, 'default_circular_splits', None)
+            option_keys = cls.default_option_keys
+            answer_key = getattr(cls, 'default_answer_key', None)
+            answer_key_switch_method = getattr(
+                cls, 'default_answer_key_switch_method', None)
+            dataset = cls.dataset_class.load(*args, **kwargs)
+            return CircularDatasetMeta.make_circular(
+                dataset,
+                circular_splits,
+                circular_patterns,
+                option_keys,
+                answer_key,
+                answer_key_switch_method,
+            )
+
+        setattr(new_cls, 'load', classmethod(load))
+        return new_cls
+
+
+class CircularCEvalDataset(CEvalDataset, metaclass=CircularDatasetMeta):
+    dataset_class = CEvalDataset
+    default_circular_splits = ['val', 'test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularMMLUDataset(MMLUDataset, metaclass=CircularDatasetMeta):
+    dataset_class = MMLUDataset
+    default_circular_splits = ['test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'target'
+
+
+class CircularCMMLUDataset(CMMLUDataset, metaclass=CircularDatasetMeta):
+    dataset_class = CMMLUDataset
+    default_circular_splits = ['test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularCSQADataset(commonsenseqaDataset, metaclass=CircularDatasetMeta):
+    dataset_class = commonsenseqaDataset
+    default_circular_splits = ['validation']
+    default_option_keys = ['A', 'B', 'C', 'D', 'E']
+    default_answer_key = 'answerKey'
+
+
+class CircularARCDataset(ARCDataset, metaclass=CircularDatasetMeta):
+    dataset_class = ARCDataset
+    default_circular_splits = None
+    default_option_keys = ['textA', 'textB', 'textC', 'textD']
+
+    def default_answer_key_switch_method(item, circular_pattern):
+        circular_pattern = tuple(i[-1] for i in circular_pattern)
+        item['answerKey'] = circular_pattern['ABCD'.index(item['answerKey'])]
+        return item
+
+
+class CircularHSWAGDataset(hellaswagDataset_V2, metaclass=CircularDatasetMeta):
+    dataset_class = hellaswagDataset_V2
+    default_circular_splits = None
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'label'
+
+
+class CircularOBQADataset(OBQADataset, metaclass=CircularDatasetMeta):
+    dataset_class = OBQADataset
+    default_circular_splits = None
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answerKey'
+
+
+class CircularRaceDataset(RaceDataset, metaclass=CircularDatasetMeta):
+    dataset_class = RaceDataset
+    default_circular_splits = ['test']
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularXiezhiDataset(XiezhiDataset, metaclass=CircularDatasetMeta):
+    dataset_class = XiezhiDataset
+    default_circular_splits = None
+    default_option_keys = ['A', 'B', 'C', 'D']
+    default_answer_key = 'answer'
+
+
+class CircularEvaluator(BaseEvaluator):
+    """This Evaluator assesses datasets post-Circular processing, generating
+    the following evaluation metrics:
+
+    - `acc_{origin|circular|all_possible}`: Treats each question with shuffled
+        answer options as separate, calculating accuracy.
+    - `perf_{origin|circular|all_possible}`: According Circular logic, a
+        question is considered correct only if all its variations with shuffled
+        options are answered correctly, calculating accuracy. perf is short for
+        perfect.
+    - `more_{num}_{origin|circular|all_possible}`: According to Circular logic,
+        a question is considered correct only if the number of its variations
+        answered correctly is greater than or equal to `num`, calculating
+        accuracy.
+
+    Note that when the `all_possible` method is used to shuffle option order,
+        it naturally includes the Circular method, and its metrics will also be
+        output.
+
+    Args:
+        circular_pattern: The method of shuffling options, either 'circular' or
+            'all_possible', defaulting to 'circular'.
+    """
+
+    def __init__(self, circular_pattern='circular'):
+        super().__init__()
+        self.circular_pattern = circular_pattern
+
+    def score(self, predictions, references, test_set):
+        circular_patterns = {}
+        circular_patterns['origin'] = get_origin_patterns(
+            test_set[0]['circular_pattern'])
+        circular_patterns['circular'] = get_circular_patterns(
+            test_set[0]['circular_pattern'])
+        if self.circular_pattern == 'all_possible':
+            circular_patterns['all_possible'] = get_all_possible_patterns(
+                test_set[0]['circular_pattern'])
+
+        metrics = {}
+        tmp_metrics = {}
+        tmp_metrics.update({f'correct_{k}': 0 for k in circular_patterns})
+        tmp_metrics.update({f'count_{k}': 0 for k in circular_patterns})
+        # calculate the original accuracy
+        for pred, ref, origin_item in zip(predictions, references, test_set):
+            circular_pattern = origin_item['circular_pattern']
+            for k in circular_patterns:
+                if tuple(circular_pattern) in circular_patterns[k]:
+                    tmp_metrics[f'correct_{k}'] += 1 if pred == ref else 0
+                    tmp_metrics[f'count_{k}'] += 1
+
+        for k in circular_patterns:
+            metrics[f'acc_{k}'] = (tmp_metrics[f'correct_{k}'] /
+                                   tmp_metrics[f'count_{k}'] * 100)
+
+        # calculate the circular accuracy
+        _details = {k: {} for k in circular_patterns}
+        for pred, ref, origin_item in zip(predictions, references, test_set):
+            index = origin_item['qid']
+            circular_pattern = origin_item['circular_pattern']
+            for k in circular_patterns:
+                if tuple(circular_pattern) in circular_patterns[k]:
+                    _details[k].setdefault(
+                        index, []).append(True if pred == ref else False)
+        for k in _details:
+            _details[k] = {
+                index: sum(_details[k][index])
+                for index in _details[k]
+            }
+        for k in _details:
+            for j in range(1, len(circular_patterns[k]) + 1):
+                count = sum([_details[k][index] >= j for index in _details[k]])
+                total = len(_details[k])
+                if j != len(circular_patterns[k]):
+                    metrics[f'more_{j}_{k}'] = count / total * 100
+                else:
+                    metrics[f'perf_{k}'] = count / total * 100
+
+        return metrics

opencompass/summarizers/__init__.py

+from .circular import CircularSummarizer
 from .default import DefaultSummarizer
 from .subjective import SubjectiveSummarizer
 
-__all__ = ['DefaultSummarizer', 'SubjectiveSummarizer']
+__all__ = ['DefaultSummarizer', 'SubjectiveSummarizer', 'CircularSummarizer']

opencompass/summarizers/circular.py

+from typing import List, Optional
+
+from mmengine import ConfigDict
+
+from opencompass.utils import dataset_abbr_from_cfg
+from opencompass.utils.prompt import get_prompt_hash
+
+from .default import DefaultSummarizer
+
+
+class CircularSummarizer(DefaultSummarizer):
+
+    def __init__(self,
+                 config: ConfigDict,
+                 dataset_abbrs: Optional[List[str]] = None,
+                 summary_groups: List = [],
+                 prompt_db=None,
+                 metric_types=None) -> None:
+        super().__init__(config, dataset_abbrs, summary_groups, prompt_db)
+        self.metric_types = metric_types
+
+    def _format_table(self, parsed_results, dataset_metrics,
+                      dataset_eval_mode):
+        prompt_version = {
+            dataset_abbr_from_cfg(d): get_prompt_hash(d)[:6]
+            for d in self.dataset_cfgs
+        }
+
+        table = []
+        header1 = ['dataset', 'version', 'mode'] + sum(
+            [[model_abbr] + ['-' for _ in range(len(self.metric_types) - 1)]
+             for model_abbr in self.model_abbrs], [])
+        table.append(header1)
+        header2 = ['-', '-', '-'] + sum(
+            [self.metric_types for _ in self.model_abbrs], [])
+        table.append(header2)
+        for dataset_abbr in self.dataset_abbrs:
+            if dataset_abbr not in dataset_metrics:
+                table.append([dataset_abbr, '-', '-'] + ['-'] *
+                             len(self.model_abbrs) * len(self.metric_types))
+                continue
+            row = [
+                dataset_abbr,
+                prompt_version.get(dataset_abbr, '-'),
+                dataset_eval_mode.get(dataset_abbr, '-')
+            ]
+            for model_abbr in self.model_abbrs:
+                for metric in self.metric_types:
+                    if dataset_abbr in parsed_results[
+                            model_abbr] and metric in parsed_results[
+                                model_abbr][dataset_abbr]:
+                        row.append('{:.02f}'.format(
+                            parsed_results[model_abbr][dataset_abbr][metric]))
+                    else:
+                        row.append('-')
+            table.append(row)
+        return table

opencompass/tasks/openicl_eval.py

@@ -172,6 +172,7 @@ class OpenICLEvalTask(BaseTask):
             preds['predictions'] = pred_strs
             preds['references'] = (test_set[self.output_column]
                                    if self.output_column else None)
+            preds['test_set'] = test_set
             preds = {
                 k: preds[k]
                 for k in signature(icl_evaluator.score).parameters