[Deperecate] Remove multi-modal related stuff (#1072)

* Remove MultiModal

* update index.rst

* update README

* remove mmbench codes

* update news

---------
Co-authored-by: Leymore <zfz-960727@163.com>

configs/multimodal/visualglm/visualglm_6b_vizwiz.py

-from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
-
-# dataloader settings
-val_pipeline = [
-    dict(type='mmpretrain.LoadImageFromFile'),
-    dict(type='mmpretrain.ToPIL', to_rgb=True),
-    dict(type='mmpretrain.torchvision/Resize',
-         size=(224, 224),
-         interpolation=3),
-    dict(type='mmpretrain.torchvision/ToTensor'),
-    dict(type='mmpretrain.torchvision/Normalize',
-         mean=(0.48145466, 0.4578275, 0.40821073),
-         std=(0.26862954, 0.26130258, 0.27577711)),
-    dict(
-        type='mmpretrain.PackInputs',
-        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
-        meta_keys=['question_id', 'image_id'],
-    )
-]
-
-dataset = dict(type='mmpretrain.VizWiz',
-               data_root='data/vizwiz/',
-               data_prefix='Images/val',
-               ann_file='Annotations/val.json',
-               pipeline=val_pipeline)
-
-visualglm_vizwiz_dataloader = dict(batch_size=1,
-                  num_workers=4,
-                  dataset=dataset,
-                  collate_fn=dict(type='pseudo_collate'),
-                  sampler=dict(type='DefaultSampler', shuffle=False))
-
-# model settings
-visualglm_vizwiz_model = dict(
-    type='visualglm',
-    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
-    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
-    post_processor=dict(type=VisualGLMBasePostProcessor)
-)
-
-# evaluation settings
-visualglm_vizwiz_evaluator = [dict(type='mmpretrain.VQAAcc')]

configs/multimodal/visualglm/visualglm_6b_vqav2.py

-from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
-
-# dataloader settings
-val_pipeline = [
-    dict(type='mmpretrain.LoadImageFromFile'),
-    dict(type='mmpretrain.ToPIL', to_rgb=True),
-    dict(type='mmpretrain.torchvision/Resize',
-         size=(224, 224),
-         interpolation=3),
-    dict(type='mmpretrain.torchvision/ToTensor'),
-    dict(type='mmpretrain.torchvision/Normalize',
-         mean=(0.48145466, 0.4578275, 0.40821073),
-         std=(0.26862954, 0.26130258, 0.27577711)),
-    dict(
-        type='mmpretrain.PackInputs',
-        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
-        meta_keys=['question_id', 'image_id'],
-    )
-]
-
-dataset = dict(
-    type='mmpretrain.COCOVQA',
-    data_root='data/coco',
-    data_prefix='images/val2014',
-    question_file='annotations/v2_OpenEnded_mscoco_val2014_questions.json',
-    ann_file='annotations/v2_mscoco_val2014_annotations.json',
-    pipeline=val_pipeline)
-
-visualglm_vqav2_dataloader = dict(batch_size=1,
-                  num_workers=4,
-                  dataset=dataset,
-                  collate_fn=dict(type='pseudo_collate'),
-                  sampler=dict(type='DefaultSampler', shuffle=False))
-
-# model settings
-visualglm_vqav2_model = dict(
-    type='visualglm',
-    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
-    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
-    post_processor=dict(type=VisualGLMBasePostProcessor)
-)
-
-# evaluation settings
-visualglm_vqav2_evaluator = [dict(type='mmpretrain.VQAAcc')]

configs/multimodal/visualglm/visualglm_6b_vsr.py

-from opencompass.multimodal.models.visualglm import (VisualGLMVSRPostProcessor, VisualGLMVQAPromptConstructor)
-
-# dataloader settings
-val_pipeline = [
-    dict(type='mmpretrain.LoadImageFromFile'),
-    dict(type='mmpretrain.ToPIL', to_rgb=True),
-    dict(type='mmpretrain.torchvision/Resize',
-         size=(224, 224),
-         interpolation=3),
-    dict(type='mmpretrain.torchvision/ToTensor'),
-    dict(type='mmpretrain.torchvision/Normalize',
-         mean=(0.48145466, 0.4578275, 0.40821073),
-         std=(0.26862954, 0.26130258, 0.27577711)),
-    dict(
-        type='mmpretrain.PackInputs',
-        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
-        meta_keys=['question_id', 'image_id'],
-    )
-]
-
-
-dataset = dict(type='mmpretrain.VSR',
-               data_root='data/vsr/',
-               data_prefix='images/',
-               ann_file='annotations/test.json',
-               pipeline=val_pipeline)
-
-visualglm_vsr_dataloader = dict(batch_size=1,
-                  num_workers=4,
-                  dataset=dataset,
-                  collate_fn=dict(type='pseudo_collate'),
-                  sampler=dict(type='DefaultSampler', shuffle=False))
-
-# model settings
-visualglm_vsr_model = dict(
-    type='visualglm',
-    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
-    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
-    post_processor=dict(type=VisualGLMVSRPostProcessor)
-)
-
-# evaluation settings
-visualglm_vsr_evaluator = [dict(type='mmpretrain.GQAAcc')]

docs/en/MMBench.md

-# Evaluation pipeline on MMBench
-
-## Intro to each data sample in MMBench
-
-MMBecnh is split into **dev** and **test** split, and each data sample in each split contains the following field:
-
-```
-img: the raw data of an image
-question: the question
-options: the concated options
-category: the leaf category
-l2-category: the l2-level category
-options_dict: the dict contains all options
-index: the unique identifier of current question
-context (optional): the context to a question, which is optional.
-answer: the target answer to current question. (only exists in the dev split, and is keep confidential for the test split on our evaluation server)
-```
-
-## Load MMBench
-
-We provide a code snippet as an example of loading MMBench
-
-```python
-import base64
-import io
-import random
-
-import pandas as pd
-from PIL import Image
-from torch.utils.data import Dataset
-
-def decode_base64_to_image(base64_string):
-    image_data = base64.b64decode(base64_string)
-    image = Image.open(io.BytesIO(image_data))
-    return image
-
-class MMBenchDataset(Dataset):
-    def __init__(self,
-                 data_file,
-                 sys_prompt='There are several options:'):
-        self.df = pd.read_csv(data_file, sep='\t')
-        self.sys_prompt = sys_prompt
-
-    def __len__(self):
-        return len(self.df)
-
-    def __getitem__(self, idx):
-        index = self.df.iloc[idx]['index']
-        image = self.df.iloc[idx]['image']
-        image = decode_base64_to_image(image)
-        question = self.df.iloc[idx]['question']
-        answer = self.df.iloc[idx]['answer'] if 'answer' in self.df.iloc[0].keys() else None
-        catetory = self.df.iloc[idx]['category']
-        l2_catetory = self.df.iloc[idx]['l2-category']
-
-        option_candidate = ['A', 'B', 'C', 'D', 'E']
-        options = {
-            cand: self.load_from_df(idx, cand)
-            for cand in option_candidate
-            if self.load_from_df(idx, cand) is not None
-        }
-        options_prompt = f'{self.sys_prompt}\n'
-        for key, item in options.items():
-            options_prompt += f'{key}. {item}\n'
-
-        hint = self.load_from_df(idx, 'hint')
-        data = {
-            'img': image,
-            'question': question,
-            'answer': answer,
-            'options': options_prompt,
-            'category': catetory,
-            'l2-category': l2_catetory,
-            'options_dict': options,
-            'index': index,
-            'context': hint,
-        }
-        return data
-   def load_from_df(self, idx, key):
-        if key in self.df.iloc[idx] and not pd.isna(self.df.iloc[idx][key]):
-            return self.df.iloc[idx][key]
-        else:
-            return None
-```
-
-## How to construct the inference prompt
-
-```python
-if data_sample['context'] is not None:
-    prompt = data_sample['context'] + ' ' + data_sample['question'] + ' ' + data_sample['options']
-else:
-    prompt = data_sample['question'] + ' ' + data_sample['options']
-```
-
-For example:
-Question: Which category does this image belong to?
-A. Oil Painting
-B. Sketch
-C. Digital art
-D. Photo
-
-<div align=center>
-<img src="https://github-production-user-asset-6210df.s3.amazonaws.com/34324155/255581681-1364ef43-bd27-4eb5-b9e5-241327b1f920.png" width="50%"/>
-</div>
-
-```python
-prompt = """
-###Human: Question: Which category does this image belong to?
-There are several options: A. Oil Painting, B. Sketch, C. Digital art, D. Photo
-###Assistant:
-"""
-```
-
-You can make custom modifications to the prompt
-
-## How to save results:
-
-You should dump your model's predictions into an excel(.xlsx) file, and this file should contain the following fields:
-
-```
-question: the question
-A: The first choice
-B: The second choice
-C: The third choice
-D: The fourth choice
-prediction: The prediction of your model to current question
-category: the leaf category
-l2_category: the l2-level category
-index: the question index
-```
-
-If there are any questions with fewer than four options, simply leave those fields blank.

docs/en/advanced_guides/multimodal_eval.md

-# Multi-modality Evaluation
-
-We support several multi-modality datasets, such as [MMBench](https://opencompass.org.cn/MMBench), [SEED-Bench](https://github.com/AILab-CVC/SEED-Bench) to evaluate multi-modality models. Before starting, please make sure you have downloaded the evaluation datasets following the official instruction.
-
-## Start Evaluation
-
-Before evaluation, you could modify `tasks.py` or create a new file like `tasks.py` to evaluate your own model.
-
-Generally to run the evaluation, we use command below.
-
-### Slurm
-
-```sh
-cd $root
-python run.py configs/multimodal/tasks.py --mm-eval --slurm -p $PARTITION
-```
-
-### PyTorch
-
-```sh
-cd $root
-python run.py configs/multimodal/tasks.py --mm-eval
-```
-
-## Configuration File
-
-We adapt the new config format of [MMEngine](https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html#a-pure-python-style-configuration-file-beta).
-
-### Task File
-
-Here is the example config of `configs/multimodal/tasks.py`.
-
-```python
-from mmengine.config import read_base
-
-with read_base():
-    from .minigpt_4.minigpt_4_7b_mmbench import (minigpt_4_mmbench_dataloader,
-                                                 minigpt_4_mmbench_evaluator,
-                                                 minigpt_4_mmbench_load_from,
-                                                 minigpt_4_mmbench_model)
-
-models = [minigpt_4_mmbench_model]
-datasets = [minigpt_4_mmbench_dataloader]
-evaluators = [minigpt_4_mmbench_evaluator]
-load_froms = [minigpt_4_mmbench_load_from]
-
-# set the platform and resources
-num_gpus = 8
-num_procs = 8
-launcher = 'pytorch'
-```
-
-### Details of Task
-
-Here is an example of MiniGPT-4 with MMBench and we provide some comments for
-users to understand the meaning of the keys in config.
-
-```python
-from opencompass.multimodal.models.minigpt_4 import (
-    MiniGPT4MMBenchPromptConstructor, MiniGPT4MMBenchPostProcessor)
-
-# dataloader settings
-# Here we use Transforms in MMPreTrain to process images
-val_pipeline = [
-    dict(type='mmpretrain.torchvision/Resize',
-         size=(224, 224),
-         interpolation=3),
-    dict(type='mmpretrain.torchvision/ToTensor'),
-    dict(type='mmpretrain.torchvision/Normalize',
-         mean=(0.48145466, 0.4578275, 0.40821073),
-         std=(0.26862954, 0.26130258, 0.27577711)),
-    dict(type='mmpretrain.PackInputs',
-         algorithm_keys=[
-             'question', 'category', 'l2-category', 'context', 'index',
-             'options_dict', 'options', 'split'
-         ])
-]
-
-# The defined MMBench datasets to load evaluation data
-dataset = dict(type='opencompass.MMBenchDataset',
-               data_file='data/mmbench/mmbench_test_20230712.tsv',
-               pipeline=val_pipeline)
-
-minigpt_4_mmbench_dataloader = dict(batch_size=1,
-                                    num_workers=4,
-                                    dataset=dataset,
-                                    collate_fn=dict(type='pseudo_collate'),
-                                    sampler=dict(type='DefaultSampler',
-                                                 shuffle=False))
-
-# model settings
-minigpt_4_mmbench_model = dict(
-    type='minigpt-4',  # the test multomodal algorithm, the type can be found in `opencompass/multimodal/models/minigpt_4.py`, `@MM_MODELS.register_module('minigpt-4')`
-    low_resource=False,
-    llama_model='/path/to/vicuna-7b/',  # the model path of LLM
-    prompt_constructor=dict(type=MiniGPT4MMBenchPromptConstructor,  # the PromptConstructor to construct the prompt
-                            image_prompt='###Human: <Img><ImageHere></Img>',
-                            reply_prompt='###Assistant:'),
-    post_processor=dict(type=MiniGPT4MMBenchPostProcessor))  # the PostProcessor to deal with the output, process it into the required format
-
-# evaluation settings
-minigpt_4_mmbench_evaluator = [
-    dict(type='opencompass.DumpResults',  # the evaluator will dump results to save_path, code can be found in `opencompass/metrics/dump_results.py`
-         save_path='work_dirs/minigpt-4-7b-mmbench.xlsx')
-]
-
-minigpt_4_mmbench_load_from = '/path/to/prerained_minigpt4_7b.pth'  # the model path of linear layer between Q-Former and LLM in MiniGPT-4
-```

docs/en/index.rst

@@ -64,7 +64,6 @@ We always welcome *PRs* and *Issues* for the betterment of OpenCompass.
    advanced_guides/evaluation_lightllm.md
    advanced_guides/code_eval.md
    advanced_guides/code_eval_service.md
-   advanced_guides/multimodal_eval.md
    advanced_guides/prompt_attack.md
    advanced_guides/longeval.md
    advanced_guides/subjective_evaluation.md

docs/en/user_guides/framework_overview.md

@@ -2,7 +2,7 @@
 
 ## Evaluation Targets
 
-The primary evaluation targets of this algorithm library are large language models and multimodal large models. We introduce specific model types for evaluation using the large language model as an example.
+The primary evaluation targets of this algorithm library are large language models. We introduce specific model types for evaluation using the large language model as an example.
 
 - base Model: Typically obtained through training on massive textual data in a self-supervised manner (e.g., OpenAI's GPT-3, Meta's LLaMA). These models usually have powerful text continuation capabilities.
 

docs/zh_cn/advanced_guides/multimodal_eval.md

-# 多模态评测
-
-我们支持了多个多模态数据集，例如 [MMBench](https://opencompass.org.cn/MMBench)，[SEED-Bench](https://github.com/AILab-CVC/SEED-Bench)，来对多模态模型进行评测。在开始评测之前，请确保您已经按照官方教程下载了评测数据集。
-
-## 开始评测
-
-在评测前，您需要先修改 `tasks.py` 或者创建一个类似的新文件 `tasks_your_model.py` 来对您的模型进行评测。
-
-一般来说我们使用下列命令启动评测。
-
-### Slurm
-
-```sh
-cd $root
-python run.py configs/multimodal/tasks.py --mm-eval --slurm -p $PARTITION
-```
-
-### PyTorch
-
-```sh
-cd $root
-python run.py configs/multimodal/tasks.py --mm-eval
-```
-
-## 配置文件
-
-We adapt the new config format of [MMEngine](https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html#a-pure-python-style-configuration-file-beta).
-
-### 任务文件
-
-这是 `configs/multimodal/tasks.py` 的示例。
-
-```python
-from mmengine.config import read_base
-
-with read_base():
-    from .minigpt_4.minigpt_4_7b_mmbench import (minigpt_4_mmbench_dataloader,
-                                                 minigpt_4_mmbench_evaluator,
-                                                 minigpt_4_mmbench_load_from,
-                                                 minigpt_4_mmbench_model)
-
-models = [minigpt_4_mmbench_model]
-datasets = [minigpt_4_mmbench_dataloader]
-evaluators = [minigpt_4_mmbench_evaluator]
-load_froms = [minigpt_4_mmbench_load_from]
-
-# set the platform and resources
-num_gpus = 8
-num_procs = 8
-launcher = 'pytorch'
-```
-
-### 细节配置
-
-这是使用 MMBench 对 MiniGPT-4 进行评测的示例，我们提供了部分注释方便用户理解配置文件的含义。
-
-```python
-from opencompass.multimodal.models.minigpt_4 import (
-    MiniGPT4MMBenchPromptConstructor, MiniGPT4MMBenchPostProcessor)
-
-# dataloader settings
-# 我们使用 MMPreTrain 中的 transforms 对图像数据进行处理
-val_pipeline = [
-    dict(type='mmpretrain.torchvision/Resize',
-         size=(224, 224),
-         interpolation=3),
-    dict(type='mmpretrain.torchvision/ToTensor'),
-    dict(type='mmpretrain.torchvision/Normalize',
-         mean=(0.48145466, 0.4578275, 0.40821073),
-         std=(0.26862954, 0.26130258, 0.27577711)),
-    dict(type='mmpretrain.PackInputs',
-         algorithm_keys=[
-             'question', 'category', 'l2-category', 'context', 'index',
-             'options_dict', 'options', 'split'
-         ])
-]
-
-# 定义 MMBench dataset 来读取对应的数据
-dataset = dict(type='opencompass.MMBenchDataset',
-               data_file='data/mmbench/mmbench_test_20230712.tsv',
-               pipeline=val_pipeline)
-
-minigpt_4_mmbench_dataloader = dict(batch_size=1,
-                                    num_workers=4,
-                                    dataset=dataset,
-                                    collate_fn=dict(type='pseudo_collate'),
-                                    sampler=dict(type='DefaultSampler',
-                                                 shuffle=False))
-
-# model settings
-minigpt_4_mmbench_model = dict(
-    type='minigpt-4',  # 被测试的多模模型，type 在 `opencompass/multimodal/models/minigpt_4.py` 的 `@MM_MODELS.register_module('minigpt-4')` 中有定义
-    low_resource=False,
-    llama_model='/path/to/vicuna-7b/',  # LLM 的模型路径
-    prompt_constructor=dict(type=MiniGPT4MMBenchPromptConstructor,  # 使用 PromptConstructor 来构建 LLM 的输入 prompt
-                            image_prompt='###Human: <Img><ImageHere></Img>',
-                            reply_prompt='###Assistant:'),
-    post_processor=dict(type=MiniGPT4MMBenchPostProcessor))  # 使用 PostProcessor 来处理模型输出，使其符合输出格式的要求
-
-# evaluation settings
-minigpt_4_mmbench_evaluator = [
-    dict(type='opencompass.DumpResults',  # evaluator 将结果保存在 save_path，代码在 `opencompass/metrics/dump_results.py`
-         save_path='work_dirs/minigpt-4-7b-mmbench.xlsx')
-]
-
-minigpt_4_mmbench_load_from = '/path/to/prerained_minigpt4_7b.pth'  # 线性层的模型路径（MiniGPT-4 中 Q-Former 和 LLM 之间的线性投影层）
-```

docs/zh_cn/index.rst

@@ -64,7 +64,6 @@ OpenCompass 上手路线
    advanced_guides/evaluation_lightllm.md
    advanced_guides/code_eval.md
    advanced_guides/code_eval_service.md
-   advanced_guides/multimodal_eval.md
    advanced_guides/prompt_attack.md
    advanced_guides/longeval.md
    advanced_guides/subjective_evaluation.md

opencompass/cli/main.py

@@ -6,13 +6,12 @@ from datetime import datetime
 
 from mmengine.config import Config, DictAction
 
-from opencompass.partitioners import MultimodalNaivePartitioner
 from opencompass.registry import PARTITIONERS, RUNNERS, build_from_cfg
 from opencompass.runners import SlurmRunner
 from opencompass.summarizers import DefaultSummarizer
 from opencompass.utils import LarkReporter, get_logger
-from opencompass.utils.run import (exec_mm_infer_runner, fill_eval_cfg,
-                                   fill_infer_cfg, get_config_from_arg)
+from opencompass.utils.run import (fill_eval_cfg, fill_infer_cfg,
+                                   get_config_from_arg)
 
 
 def parse_args():
@@ -34,11 +33,6 @@ def parse_args():
                                help='Whether to force tasks to run on dlc. If '
                                'True, `--aliyun-cfg` must be set. Defaults'
                                ' to False')
-    # multi-modal support
-    parser.add_argument('--mm-eval',
-                        help='Whether or not enable multimodal evaluation',
-                        action='store_true',
-                        default=False)
     # Add shortcut parameters (models, datasets and summarizer)
     parser.add_argument('--models', nargs='+', help='', default=None)
     parser.add_argument('--datasets', nargs='+', help='', default=None)
@@ -278,13 +272,6 @@ def main():
                            'also specified --slurm or --dlc. '
                            'The "infer" configuration will be overridden by '
                            'your runtime arguments.')
-        # Check whether run multimodal evaluation
-        if args.mm_eval:
-            partitioner = MultimodalNaivePartitioner(
-                osp.join(cfg['work_dir'], 'predictions/'))
-            tasks = partitioner(cfg)
-            exec_mm_infer_runner(tasks, args, cfg)
-            return
 
         if args.dlc or args.slurm or cfg.get('infer', None) is None:
             fill_infer_cfg(cfg, args)

opencompass/multimodal/datasets/__init__.py

-from .mmbench import MMBenchDataset  # noqa: F401, F403
-from .mme import MMEDataset  # noqa: F401, F403
-from .seedbench import SEEDBenchDataset  # noqa: F401, F403
-
-__all__ = ['MMBenchDataset'
-           'SEEDBenchDataset', 'MMEDataset']

opencompass/multimodal/datasets/mmbench.py

-import base64
-import io
-from typing import List, Optional
-
-import pandas as pd
-from mmengine.dataset import Compose
-from PIL import Image
-from torch.utils.data import Dataset
-
-from opencompass.registry import DATASETS
-
-
-def decode_base64_to_image(base64_string) -> Image:
-    """Convert raw data into Pillow image."""
-    image_data = base64.b64decode(base64_string)
-    image = Image.open(io.BytesIO(image_data))
-    return image
-
-
-@DATASETS.register_module()
-class MMBenchDataset(Dataset):
-    """Dataset to load MMBench dataset.
-
-    Args:
-        data_file (str): The path of the dataset.
-        pipeline (dict): The data augmentation.
-        sys_prompt (str): The system prompt added to the head
-            of these options. Defaults to
-            There are several options:
-    """
-
-    def __init__(self,
-                 data_file: str,
-                 pipeline: List[dict],
-                 sys_prompt: str = 'There are several options:') -> None:
-        self.df = pd.read_csv(data_file, sep='\t')
-        self.pipeline = Compose(pipeline)
-        self.sys_prompt = sys_prompt
-
-    def __len__(self) -> None:
-        return len(self.df)
-
-    def __getitem__(self, idx: int) -> dict:
-        # Mandatory Fields Begin
-        index = self.df.iloc[idx]['index']
-        image = self.df.iloc[idx]['image']
-        image = decode_base64_to_image(image)
-        question = self.df.iloc[idx]['question']
-
-        option_candidate = ['A', 'B', 'C', 'D', 'E']
-        options = {
-            cand: self.load_from_df(idx, cand)
-            for cand in option_candidate
-            if self.load_from_df(idx, cand) is not None
-        }
-        options_prompt = f'{self.sys_prompt}\n'
-        for key, item in options.items():
-            options_prompt += f'{key}. {item}\n'
-        # Mandatory Fields End
-
-        # Optional Fields Begin
-        hint = self.load_from_df(idx, 'hint')
-        category = self.load_from_df(idx, 'category')
-        l2_catetory = self.load_from_df(idx, 'l2-category')
-        # Optional Fields End
-
-        data = {
-            'img': image,
-            'question': question,
-            'options': options_prompt,
-            'category': category,
-            'l2-category': l2_catetory,
-            'options_dict': options,
-            'index': index,
-            'context': hint,
-        }
-        data = self.pipeline(data)
-        return data
-
-    def load_from_df(self, idx: int, key: str) -> Optional[str]:
-        if key in self.df.iloc[idx] and not pd.isna(self.df.iloc[idx][key]):
-            return self.df.iloc[idx][key]
-        else:
-            return None

opencompass/multimodal/datasets/mme.py

-import os
-from typing import List
-
-from mmengine.dataset import Compose
-from torch.utils.data import Dataset
-
-from opencompass.registry import DATASETS
-
-
-@DATASETS.register_module()
-class MMEDataset(Dataset):
-    """Dataset to load MME dataset.
-
-    Args:
-        data_dir (str): The path of the dataset.
-        pipeline (List[dict]): The data augmentation.
-    """
-    tasks = [
-        'artwork', 'celebrity', 'code_reasoning', 'color',
-        'commonsense_reasoning', 'count', 'existence', 'landmark',
-        'numerical_calculation', 'OCR', 'position', 'posters', 'scene',
-        'text_translation'
-    ]
-    sub_dir_name = ('images', 'questions_answers_YN')
-
-    def __init__(self, data_dir: str, pipeline: List[dict]) -> None:
-        self.pipeline = Compose(pipeline)
-        self.load_data(data_dir)
-
-    def load_data(self, data_dir: str):
-        self.data_list = []
-        image_dir, question_dir = self.sub_dir_name
-        for task in self.tasks:
-            if os.path.exists(os.path.join(data_dir, task, question_dir)):
-                q_list = os.listdir(os.path.join(data_dir, task, question_dir))
-                i_list = os.listdir(os.path.join(data_dir, task, image_dir))
-                q_prefix = os.path.join(data_dir, task, question_dir)
-                i_prefix = os.path.join(data_dir, task, image_dir)
-            else:
-                fn_list = os.listdir(os.path.join(data_dir, task))
-                q_list = [fn for fn in fn_list if '.txt' in fn]
-                i_list = [fn for fn in fn_list if fn not in q_list]
-                q_prefix = i_prefix = os.path.join(data_dir, task)
-
-            q_list.sort()
-            i_list.sort()
-            assert len(q_list) == len(i_list)
-            for q_fn, i_fn in zip(q_list, i_list):
-                assert q_fn.split('.')[0] == i_fn.split('.')[0]
-                q_path = os.path.join(q_prefix, q_fn)
-                image_path = os.path.join(i_prefix, i_fn)
-                with open(q_path, 'r') as f:
-                    q1, a1 = f.readline().strip().split('\t')
-                    q2, a2 = f.readline().strip().split('\t')
-                self.data_list.append({
-                    'img_path': image_path,
-                    'question': q1,
-                    'answer': a1,
-                    'task': task
-                })
-                self.data_list.append({
-                    'img_path': image_path,
-                    'question': q2,
-                    'answer': a2,
-                    'task': task
-                })
-
-    def __len__(self) -> None:
-        return len(self.data_list)
-
-    def __getitem__(self, idx: int) -> dict:
-        data_sample = self.data_list[idx]
-        data_sample = self.pipeline(data_sample)
-        return data_sample

opencompass/multimodal/datasets/seedbench.py

-import importlib
-import json
-import os.path as osp
-from typing import List
-
-import numpy as np
-import torch
-from decord import VideoReader, cpu
-from mmengine.dataset import Compose
-from PIL import Image
-from torch.utils.data import Dataset
-
-from opencompass.registry import DATASETS
-
-
-@DATASETS.register_module()
-class SEEDBenchDataset(Dataset):
-    """Dataset to load SEED-Bench dataset.
-
-    Args:
-        ann_file (str): The path of the annotation file.
-        cc3m_path (str): The data path of the image dimension(1-9).
-        sthv2_path (str): The data path of the dimension 10.
-        epic_kitchens_path (str): The data path of the dimension 11.
-        breakfast_path (str): The data path of the dimension 12.
-        image_pipeline (List[dict]): The data transforms for image.
-        video_pipeline (List[dict]): The data transforms for video.
-        only_image (bool): Whether run SEED-Bench only with image data.
-            Defaults to True.
-    """
-
-    def __init__(
-        self,
-        ann_file: str,
-        cc3m_path: str,
-        sthv2_path: str,
-        epic_kitchens_path: str,
-        breakfast_path: str,
-        image_pipeline: List[dict],
-        video_pipeline: List[dict],
-        only_image: bool = True,
-    ) -> None:
-        ann_file = json.load(open(ann_file, 'rb'))
-        if 'questions' in ann_file.keys():
-            self.ann_file = ann_file['questions']
-        self.cc3m_path = cc3m_path
-        self.sthv2_path = sthv2_path
-        self.epic_kitchens_path = epic_kitchens_path
-        self.breakfast_path = breakfast_path
-        self.image_pipeline = Compose(image_pipeline)
-        if only_image:
-            image_ann_file = [
-                ann for ann in self.ann_file if ann['data_type'] == 'image'
-            ]
-            self.ann_file = image_ann_file
-        if not only_image:
-            raise NotImplementedError
-            self.video_pipeline = Compose(video_pipeline)
-
-    def __len__(self) -> None:
-        return len(self.ann_file)
-
-    def __getitem__(self, idx: str) -> dict:
-        item = self.ann_file[idx]
-        data = {
-            'question':
-            item['question'],
-            'answer':
-            item['answer'],
-            'choices': [
-                item['choice_a'], item['choice_b'], item['choice_c'],
-                item['choice_d']
-            ],
-            'data_type':
-            item['data_type'],
-            'question_id':
-            item['question_id'],
-            'question_type_id':
-            item['question_type_id'],
-            'index':
-            idx,
-        }
-
-        if item['data_type'] == 'image':
-            data_path = osp.join(self.cc3m_path, item['data_id'])
-            raw_image = Image.open(open(data_path, 'rb')).convert('RGB')
-            data['data_path'] = data_path
-            data['img'] = raw_image
-            data = self.image_pipeline(data)
-        elif item['data_type'] == 'video':
-            if item['question_type_id'] == 10:
-                data_path = osp.join(self.sthv2_path, item['data_id'])
-                data['data_path'] = data_path
-            elif item['question_type_id'] == 11:
-                data_path = osp.join(self.epic_kitchens_path, item['data_id'])
-                data['data_path'] = data_path
-                data['segment'] = item['segment']
-            elif item['question_type_id'] == 12:
-                data_path = osp.join(self.breakfast_path, item['data_id'])
-                data['data_path'] = data_path
-                data['segment'] = item['segment']
-            else:
-                raise ValueError('The question type id is not valid.')
-
-            # preprocessing videos in evaluation dimension 10-12
-            use_pyav = False
-            if 'segment' in data.keys():
-                segment = data['segment']
-                if isinstance(segment[0], int):
-                    # using pyav for decoding videos in evaluation dimension 12
-                    use_pyav = True
-                start, end = segment[0], segment[1]
-            else:
-                start = 0.0
-                end = 0.0
-
-            if use_pyav:
-                # using pyav for videos in evaluation dimension 12
-                av = importlib.importmodule('av')
-                reader = av.open(data_path)
-                frames = [
-                    torch.from_numpy(f.to_rgb().to_ndarray())
-                    for f in reader.decode(video=0)
-                ]
-                video_len = len(frames)
-                start_frame, end_frame = start, end
-                end_frame = min(end_frame, video_len)
-                offset = self.get_index(end_frame - start_frame, 8)
-                frame_indices = offset + start_frame
-                buffer = torch.stack([frames[idx] for idx in frame_indices])
-                buffer = buffer.numpy()
-            else:
-                # using decord for videos in evaluating dimension 10-11
-                import io
-
-                import mmengine.fileio as fileio
-                file_obj = io.BytesIO(fileio.get(data_path))
-                vr = VideoReader(file_obj, num_threads=1, ctx=cpu(0))
-                video_len = len(vr)
-                fps = vr.get_avg_fps()
-                if 'segment' in data.keys():
-                    # obtain start and end frame for the video segment
-                    # in evaluation dimension 11
-                    start_frame = int(min(max(start * fps, 0), video_len - 1))
-                    end_frame = int(min(max(end * fps, 0), video_len - 1))
-                    tot_frames = int(end_frame - start_frame)
-                    offset = self.get_index(tot_frames, 8)
-                    frame_indices = offset + start_frame
-                else:
-                    # sample frames of the video in evaluation dimension 10
-                    frame_indices = self.get_index(video_len - 1, 8)
-                vr.seek(0)
-                buffer = vr.get_batch(frame_indices)
-                buffer = buffer.asnumpy()
-            data['imgs'] = buffer
-            data = self.video_pipeline(data)
-
-        else:
-            raise ValueError('The data type is not valid.')
-
-        return data
-
-    def get_index(self, num_frames, num_segments):
-        if num_segments > num_frames:
-            offsets = np.array([idx for idx in range(num_frames)])
-        else:
-            # uniform sampling
-            seg_size = float(num_frames - 1) / num_segments
-            start = int(seg_size / 2)
-            offsets = np.array([
-                start + int(np.round(seg_size * idx))
-                for idx in range(num_segments)
-            ])
-        return offsets

opencompass/multimodal/models/__init__.py

-import os.path as osp
-
-from opencompass.utils import satisfy_requirement
-
-if satisfy_requirement('salesforce-lavis'):
-    from .instructblip import *  # noqa: F401, F403
-
-if osp.exists('opencompass/multimodal/models/minigpt_4/MiniGPT-4'):
-    from .minigpt_4 import *  # noqa: F401, F403
-
-if osp.exists(
-        'opencompass/multimodal/models/llama_adapter_v2_multimodal/LLaMA-Adapter'  # noqa
-):
-    from .llama_adapter_v2_multimodal import *  # noqa: F401, F403
-
-from .llava import *  # noqa: F401, F403
-
-if osp.exists('opencompass/multimodal/models/mplug_owl/mPLUG-Owl'):
-    from .mplug_owl import *  # noqa: F401, F403
-
-from .openflamingo import *  # noqa: F401, F403
-from .otter import *  # noqa: F401, F403
-from .qwen import *  # noqa: F401, F403
-from .visualglm import *  # noqa: F401, F403

opencompass/multimodal/models/instructblip/__init__.py

-from .blip2_vicuna_instruct import InstructBlipInferencer
-from .post_processor import (InstructBlipCOCOCaptionPostProcessor,
-                             InstructBlipMMBenchPostProcessor,
-                             InstructBlipScienceQAPostProcessor,
-                             InstructBlipVQAPostProcessor,
-                             InstructBlipVSRPostProcessor)
-from .prompt_constructor import (InstructBlipCOCOCaotionPromptConstructor,
-                                 InstructBlipMMBenchPromptConstructor,
-                                 InstructBlipScienceQAPromptConstructor,
-                                 InstructBlipVQAPromptConstructor,
-                                 InstructBlipVSRPromptConstructor)
-
-__all__ = [
-    'InstructBlipInferencer',
-    'InstructBlipMMBenchPromptConstructor',
-    'InstructBlipMMBenchPostProcessor',
-    'InstructBlipCOCOCaotionPromptConstructor',
-    'InstructBlipCOCOCaptionPostProcessor',
-    'InstructBlipVQAPromptConstructor',
-    'InstructBlipVQAPostProcessor',
-    'InstructBlipScienceQAPromptConstructor',
-    'InstructBlipScienceQAPostProcessor',
-    'InstructBlipVSRPromptConstructor',
-    'InstructBlipVSRPostProcessor',
-]

opencompass/multimodal/models/instructblip/post_processor.py

-import random
-import re
-
-import torch
-
-
-class InstructBlipMMBenchPostProcessor:
-    """"Post processor for MiniGPT-4 on MMBench."""
-
-    def __init__(self) -> None:
-        pass
-
-    def __call__(self, output_token: torch.tensor, tokenizer) -> str:
-        # convert output id 0 to 2 (eos_token_id)
-        output_token[output_token == 0] = 2
-        output_text = tokenizer.decode(output_token,
-                                       add_special_tokens=False)  # noqa
-        output_text = self._extract_key_words(output_text.strip())
-        return output_text
-
-    def _extract_key_words(self, output_text: str) -> str:
-
-        output_text = output_text.split('###')[0]
-        output_text = output_text.split('Assistant:')[-1].strip()
-        output_text = output_text.strip('</s><s>')
-        output_text = output_text.strip('</Img>')
-        output_text = output_text.strip()
-        pattern = re.compile(r'([A-Z]\.)')
-        res = pattern.findall(output_text)
-        if len(res) > 0:
-            output_text = res[0][:-1]
-        return output_text
-
-
-class InstructBlipCOCOCaptionPostProcessor:
-    """"Post processor for InstructBlip on COCO Caption."""
-
-    def __init__(self) -> None:
-        pass
-
-    def __call__(self, output_token: torch.tensor, tokenizer) -> str:
-
-        output_token[output_token == 0] = 2
-        output_text = tokenizer.decode(output_token,
-                                       add_special_tokens=False)  # noqa
-        output_text = output_text.split('###')[0]
-        output_text = output_text.split('Assistant:')[-1].strip()
-        output_text = output_text.strip('</s><s>')
-        output_text = output_text.strip('</Img>')
-        output_text = output_text.strip()
-        return output_text
-
-
-class InstructBlipVQAPostProcessor:
-    """"Post processor for InstructBlip on VQA."""
-
-    def __init__(self) -> None:
-        pass
-
-    def __call__(self, output_token: torch.tensor, tokenizer) -> str:
-        output_token[output_token == 0] = 2
-        output_text = tokenizer.decode(output_token,
-                                       add_special_tokens=False)  # noqa
-        output_text = output_text.split('###')[0]
-        output_text = output_text.split('Assistant:')[-1].strip()
-        output_text = output_text.strip('</s><s>')
-        output_text = output_text.strip('</Img>')
-        output_text = output_text.strip()
-        return output_text
-
-
-class InstructBlipScienceQAPostProcessor:
-    """"Post processor for InstructBlip on ScienceQA."""
-
-    def __init__(self) -> None:
-        pass
-
-    def __call__(self, output_token: torch.tensor, tokenizer) -> str:
-
-        output_token[output_token == 0] = 2
-        output_text = tokenizer.decode(output_token,
-                                       add_special_tokens=False)  # noqa
-        output_text = output_text.split('###')[0]
-        output_text = output_text.split('Assistant:')[-1].strip()
-        output_text = output_text.strip('</s><s>')
-        output_text = output_text.strip('</Img>')
-        output_text = output_text.strip()
-        pattern = re.compile(r'\(([A-Z])\)')
-        output_text = pattern.findall(output_text)
-        if len(output_text) == 0:
-            output_text = random.choice(['A', 'B', 'C', 'D'])
-        else:
-            output_text = output_text[0]
-        return output_text
-
-
-class InstructBlipVSRPostProcessor:
-    """"Post processor for InstructBlip on VSR."""
-
-    def __init__(self) -> None:
-        pass
-
-    def __call__(self, output_token: torch.tensor, tokenizer) -> str:
-
-        output_token[output_token == 0] = 2
-        output_text = tokenizer.decode(output_token, add_special_tokens=False)
-        pattern = r'yes|no|Yes|No'
-        output_text = re.findall(pattern, output_text)
-        if len(output_text) > 0:
-            output_text = output_text[0].lower()
-        return output_text

opencompass/multimodal/models/llama_adapter_v2_multimodal/__init__.py

-from .llama_adapter import LLaMA_adapter_v2
-from .post_processor import LlamaAadapterMMBenchPostProcessor
-from .prompt_constructor import LlamaAadapterMMBenchPromptConstructor  # noqa
-
-__all__ = [
-    'LLaMA_adapter_v2', 'LlamaAadapterMMBenchPostProcessor',
-    'LlamaAadapterMMBenchPromptConstructor'
-]