[Feature]: Use multimodal (#73)

* [Feature]: Add minigpt-4

* [Feature]: Add mm local runner

* [Feature]: Add instructblip

* [Feature]: Delete redundant file

* [Feature]: Delete redundant file

* [Feature]: Add README to InstructBLIP

* [Feature]: Update MiniGPT-4

* [Fix]: Fix lint

* [Feature]add omnibenchmark readme (#49)

* add omnibenchmark readme

* fix

* Update OmniMMBench.md

* Update OmniMMBench.md

* Update OmniMMBench.md

* [Fix]: Refine name (#54)

* [Feature]: Unify out and err

* [Fix]: Fix lint

* [Feature]: Rename to mmbench and change weight path

* [Feature]: Delete Omni in instructblip

* [Feature]: Check the avaliablity of lavis

* [Fix]: Fix lint

* [Feature]: Refactor MM

* [Refactor]: Refactor path

* [Feature]: Delete redundant files

* [Refactor]: Delete redundant files

---------
Co-authored-by: Wangbo Zhao(黑色枷锁) <56866854+wangbo-zhao@users.noreply.github.com>

opencompass/registry.py

+from mmengine.registry import DATASETS as MMENGINE_DATASETS
+from mmengine.registry import METRICS as MMENGINE_METRICS
+from mmengine.registry import MODELS as MMENGINE_MODELS
 from mmengine.registry import Registry
 
 PARTITIONERS = Registry('partitioner', locations=['opencompass.partitioners'])
@@ -22,3 +25,12 @@ ICL_PROMPT_TEMPLATES = Registry(
     locations=['opencompass.openicl.icl_prompt_template'])
 ICL_EVALUATORS = Registry('icl_evaluators',
                           locations=['opencompass.openicl.icl_evaluator'])
+DATASETS = Registry('mm_datasets',
+                    parent=MMENGINE_DATASETS,
+                    locations=['opencompass.multimodal.datasets'])
+METRICS = Registry('metric',
+                   parent=MMENGINE_METRICS,
+                   locations=['opencompass.metrics'])
+MM_MODELS = Registry('mm_model',
+                     parent=MMENGINE_MODELS,
+                     locations=['opencompass.multimodal.models'])

opencompass/runners/slurm.py

@@ -81,7 +81,6 @@ class SlurmRunner(BaseRunner):
         Returns:
             tuple[str, int]: Task name and exit code.
         """
-
         task_type = self.task_cfg.type
         if isinstance(self.task_cfg.type, str):
             task_type = TASKS.get(task_type)

opencompass/tasks/__init__.py

+from .mm_infer import *  # noqa: F401, F403
 from .openicl_eval import *  # noqa: F401, F403
 from .openicl_infer import *  # noqa: F401, F403

opencompass/tasks/mm_infer.py

+import argparse
+import json
+import os
+import os.path as osp
+import random
+import time
+from typing import Sequence
+
+import torch
+import torch.distributed as dist
+from mmengine.config import Config, ConfigDict
+from mmengine.device import get_device
+from mmengine.dist import init_dist
+from mmengine.evaluator import Evaluator
+from mmengine.logging import print_log
+from mmengine.model.wrappers import MMDistributedDataParallel
+from mmengine.runner import Runner
+from mmengine.utils import track_iter_progress
+
+from opencompass.registry import MM_MODELS, TASKS
+from opencompass.utils import get_logger
+
+
+def build_model(cfg):
+    model = MM_MODELS.build(cfg['model'])
+    load_from = cfg.get('load_from', None)
+    if load_from is not None:
+        state_dict = torch.load(cfg['load_from'], map_location='cpu')
+        if 'model' in state_dict:
+            state_dict = state_dict['model']
+        elif 'state_dict' in state_dict:
+            state_dict = state_dict['state_dict']
+        msg = model.load_state_dict(state_dict, strict=False)
+        print_log(msg)
+    model.to(get_device())
+    if dist.is_initialized():
+        model = MMDistributedDataParallel(
+            model,
+            device_ids=[int(os.environ['LOCAL_RANK'])],
+            broadcast_buffers=False)
+    return model
+
+
+@TASKS.register_module(force=(__name__ == '__main__'))  # A hack for script run
+class MultimodalInferTask:
+    """Multimodal Inference Task.
+
+    This task is used to run the inference process.
+    """
+
+    def __init__(self, cfg: ConfigDict):
+        self.num_gpus = cfg.get('num_gpus', 0)
+        self.num_procs = cfg.get('num_procs', 1)
+        self.dataloader = cfg.get('dataset')
+        self.model = cfg.get('model')
+        self.evaluator = cfg.get('evaluator')
+        self.cfg = cfg
+        self.logger = get_logger()
+
+    @property
+    def name(self) -> str:
+        model_name = self.model['type']
+        dataset_name = self.dataloader['dataset']['type']
+        evaluator_name = self.evaluator[0]['type']
+        return f'{model_name}-{dataset_name}-{evaluator_name}'
+
+    def get_command(self, cfg_path, template):
+        """Get the command template for the task.
+
+        Args:
+            cfg_path (str): The path to the config file of the task.
+            template (str): The template which have '{task_cmd}' to format
+                the command.
+        """
+        script_path = __file__
+        if self.num_gpus > 0:
+            port = random.randint(12000, 32000)
+            command = (f'torchrun --master_port={port} '
+                       f'--nproc_per_node {self.num_procs} '
+                       f'{script_path} {cfg_path}')
+        else:
+            command = f'python {script_path} {cfg_path}'
+
+        return template.format(task_cmd=command)
+
+    def run(self):
+        # only support slurm, pytorch, mpi
+        init_dist(self.cfg.launcher)
+        self.logger.info(f'Task {self.name}')
+        # build dataloader
+        dataloader = Runner.build_dataloader(self.dataloader)
+        # build model
+        model = build_model(self.cfg)
+        # build evaluator
+        evaluator = Evaluator(self.evaluator)
+
+        for batch in track_iter_progress(dataloader):
+            if dist.is_initialized():
+                data_samples = model.module.generate(batch)
+            else:
+                data_samples = model.generate(batch)
+            if not isinstance(data_samples, Sequence):
+                data_samples = [data_samples]
+            evaluator.process(data_samples)
+
+        metrics = evaluator.evaluate(len(dataloader.dataset))
+        metrics_file = osp.join(cfg.work_dir, 'res.log')
+        with open(metrics_file, 'w') as f:
+            json.dump(metrics, f)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Model Inferencer')
+    parser.add_argument('config', help='Config file path')
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    cfg = Config.fromfile(args.config)
+    start_time = time.time()
+    inferencer = MultimodalInferTask(cfg)
+    inferencer.run()
+    end_time = time.time()
+    get_logger().info(f'time elapsed: {end_time - start_time:.2f}s')

opencompass/utils/__init__.py

 from .abbr import *  # noqa
 from .build import *  # noqa
 from .collect_env import *  # noqa
+from .dependency import *  # noqa
 from .fileio import *  # noqa
 from .git import *  # noqa
 from .lark import *  # noqa

opencompass/utils/dependency.py

+import re
+
+from importlib_metadata import PackageNotFoundError, distribution
+from mmengine.utils import digit_version
+
+
+def satisfy_requirement(dep):
+    pat = '(' + '|'.join(['>=', '==', '>']) + ')'
+    parts = re.split(pat, dep, maxsplit=1)
+    parts = [p.strip() for p in parts]
+    package = parts[0]
+    if len(parts) > 1:
+        op, version = parts[1:]
+        op = {
+            '>=': '__ge__',
+            '==': '__eq__',
+            '>': '__gt__',
+            '<': '__lt__',
+            '<=': '__le__'
+        }[op]
+    else:
+        op, version = None, None
+
+    try:
+        dist = distribution(package)
+        if op is None or getattr(digit_version(dist.version), op)(
+                digit_version(version)):
+            return True
+    except PackageNotFoundError:
+        pass
+
+    return False

run.py

@@ -6,7 +6,8 @@ from datetime import datetime
 
 from mmengine.config import Config
 
-from opencompass.partitioners import NaivePartitioner, SizePartitioner
+from opencompass.partitioners import (MultimodalNaivePartitioner,
+                                      NaivePartitioner, SizePartitioner)
 from opencompass.registry import PARTITIONERS, RUNNERS
 from opencompass.runners import DLCRunner, LocalRunner, SlurmRunner
 from opencompass.utils import LarkReporter, Summarizer, get_logger
@@ -37,6 +38,10 @@ def parse_args():
                         'redirected to files',
                         action='store_true',
                         default=False)
+    parser.add_argument('--mm-eval',
+                        help='Whether or not enable multimodal evaluation',
+                        action='store_true',
+                        default=False)
     parser.add_argument('--dry-run',
                         help='Dry run mode, in which the scheduler will not '
                         'actually run the tasks, but only print the commands '
@@ -201,7 +206,14 @@ def main():
                            'also specified --slurm or --dlc. '
                            'The "infer" configuration will be overridden by '
                            'your runtime arguments.')
-        if args.dlc or args.slurm or cfg.get('infer', None) is None:
+        # Check whether run multimodal evaluation
+        if args.mm_eval:
+            partitioner = MultimodalNaivePartitioner(
+                osp.join(cfg['work_dir'], 'predictions/'))
+            tasks = partitioner(cfg)
+            exec_mm_infer_runner(tasks, args, cfg)
+            return
+        elif args.dlc or args.slurm or cfg.get('infer', None) is None:
             # Use SizePartitioner to split into subtasks
             partitioner = SizePartitioner(
                 osp.join(cfg['work_dir'], 'predictions/'),
@@ -283,6 +295,27 @@ def main():
         summarizer.summarize(time_str=cfg_time_str)
 
 
+def exec_mm_infer_runner(tasks, args, cfg):
+    """execute multimodal infer runner according to args."""
+    if args.slurm:
+        runner = SlurmRunner(dict(type='MultimodalInferTask'),
+                             max_num_workers=args.max_num_workers,
+                             partition=args.partition,
+                             quotatype=args.quotatype,
+                             retry=args.retry,
+                             debug=args.debug,
+                             lark_bot_url=cfg['lark_bot_url'])
+    elif args.dlc:
+        raise NotImplementedError('Currently, we do not support evaluating \
+                             multimodal models on dlc.')
+    else:
+        runner = LocalRunner(task=dict(type='MultimodalInferTask'),
+                             max_num_workers=args.max_num_workers,
+                             debug=args.debug,
+                             lark_bot_url=cfg['lark_bot_url'])
+    runner(tasks)
+
+
 def exec_infer_runner(tasks, args, cfg):
     """execute infer runner according to args."""
     if args.slurm: