eval_with_model_dataset_combinations.py

from mmengine.config import read_base

with read_base():
    from opencompass.configs.datasets.ceval.ceval_gen_5f30c7 import \
        ceval_datasets as chat_ceval_datasets
    from opencompass.configs.datasets.ceval.ceval_ppl_578f8d import \
        ceval_datasets as base_ceval_datasets
    from opencompass.configs.internal.clusters.slurm import eval, infer
    from opencompass.configs.models.qwen.hf_qwen_7b import \
        models as hf_qwen_7b_base_models
    from opencompass.configs.models.qwen.hf_qwen_7b_chat import \
        models as hf_qwen_7b_chat_models

    # from opencompass.configs.internal.clusters.slurm import infer_split as infer, eval
    # from opencompass.configs.internal.clusters.slurm import infer_size as infer, eval
    # from opencompass.configs.internal.clusters.slurm import infer_size_split as infer, eval

base_ceval_datasets = base_ceval_datasets[:1]
chat_ceval_datasets = chat_ceval_datasets[-1:]

# If you do not want to run all the combinations of models and datasets, you
# can specify the combinations you want to run here. This is useful when you
# deleberately want to skip some subset of the combinations.
# Models and datasets in different combinations are recommended to be disjoint
# (different `abbr` in model & dataset configs), as we haven't tested this case
# throughly.
model_dataset_combinations = [
    dict(models=hf_qwen_7b_base_models, datasets=base_ceval_datasets),
    dict(models=hf_qwen_7b_chat_models, datasets=chat_ceval_datasets),
    # dict(models=[model_cfg1, ...], datasets=[dataset_cfg1, ...]),
]

# This union of models and datasets in model_dataset_combinations should be
# stored in the `models` and `datasets` variables below. Otherwise, modules
# like summarizer will miss out some information.
models = [*hf_qwen_7b_base_models, *hf_qwen_7b_chat_models]
datasets = [*base_ceval_datasets, *chat_ceval_datasets]

work_dir = './outputs/default/mdcomb/'
"""
dataset                 version    metric    mode    qwen-7b-hf    qwen-7b-chat-hf
----------------------  ---------  --------  ------  ------------  -----------------
ceval-computer_network  9b9417     accuracy  ppl     52.63         -
ceval-physician         6e277d     accuracy  gen     -             59.18
"""