[Feat] Support public dataset of visualglm and llava. (#265)

* [Feat] Add public dataset support of VisualGLM.

* [Feat] Refactor LLaVA.

* [Feat] Add public dataset support of LlaVA.

* [Fix] Add  arg.

configs/multimodal/llava/llava_7b_coco_caption.py

+from opencompass.multimodal.models.llava import LLaVABasePromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(type='mmpretrain.PackInputs', algorithm_keys=['image_id']),
+]
+
+
+dataset = dict(type='mmpretrain.COCOCaption',
+               data_root='data/coco',
+               data_prefix=dict(img_path='images'),
+               ann_file='annotations/coco_karpathy_val.json',
+               pipeline=val_pipeline)
+
+llava_coco_caption_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_coco_caption_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    is_caption_task=True,
+    prompt_constructor=dict(type=LLaVABasePromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_coco_caption_evaluator = [
+    dict(
+        type='mmpretrain.COCOCaption',
+        ann_file='data/coco/annotations/coco_karpathy_val_gt.json',
+    )  # noqa
+]
+

configs/multimodal/llava/llava_7b_flickr30k.py

+from opencompass.multimodal.models.llava import LLaVABasePromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(type='mmpretrain.PackInputs', algorithm_keys=['image_id']),
+]
+
+
+dataset = dict(type='mmpretrain.Flickr30kCaption',
+               data_root='data/flickr30k',
+               ann_file='annotations/dataset_flickr30k.json',
+               data_prefix='images',
+               split='val',
+               pipeline=val_pipeline)
+
+llava_flickr30k_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_flickr30k_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    is_caption_task=True,
+    prompt_constructor=dict(type=LLaVABasePromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_flickr30k_evaluator = [
+    dict(
+        type='mmpretrain.COCOCaption',
+        ann_file='data/flickr30k/annotations/flickr30k_val_gt.json',
+    )  # noqa
+]
+
+

configs/multimodal/llava/llava_7b_gqa.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+
+dataset = dict(type='mmpretrain.GQA',
+               data_root='data/gqa',
+               data_prefix='images',
+               ann_file='annotations/testdev_balanced_questions.json',
+               pipeline=val_pipeline)
+
+llava_gqa_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_gqa_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_gqa_evaluator = [dict(type='mmpretrain.GQAAcc')]
+
+

configs/multimodal/llava/llava_7b_mmbench.py

+from opencompass.multimodal.models.llava import LLaVAMMBenchPromptConstructor, LLaVABasePostProcessor
+
 # dataloader settings
 val_pipeline = [
     dict(type='mmpretrain.torchvision/Resize',
@@ -34,6 +36,8 @@ mmbench_dataloader = dict(
 llava_model = dict(
     type='llava',
     model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAMMBenchPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
 )  # noqa
 
 # evaluation settings

configs/multimodal/llava/llava_7b_ocr_vqa.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(type='mmpretrain.OCRVQA',
+               data_root='data/ocrvqa',
+               ann_file='annotations/dataset.json',
+               split='test',
+               data_prefix='images',
+               pipeline=val_pipeline)
+
+llava_ocrvqa_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_ocrvqa_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_ocrvqa_evaluator = [dict(type='mmpretrain.VQAAcc')]
+
+

configs/multimodal/llava/llava_7b_ok_vqa.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(
+    type='mmpretrain.COCOVQA',
+    data_root='data/okvqa',
+    question_file='annotations/OpenEnded_mscoco_val2014_questions.json',
+    ann_file='annotations/mscoco_val2014_annotations.json',
+    pipeline=val_pipeline,
+    data_prefix='images/val2014',
+)
+
+llava_okvqa_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_okvqa_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_okvqa_evaluator = [dict(type='mmpretrain.VQAAcc')]
+
+

configs/multimodal/llava/llava_7b_scienceqa.py

+from opencompass.multimodal.models.llava import LLaVAScienceQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(type='mmpretrain.PackInputs',
+         algorithm_keys=[
+             'question', 'gt_answer', 'choices', 'hint', 'lecture', 'solution'
+         ])
+]
+
+dataset = dict(type='mmpretrain.ScienceQA',
+               data_root='./data/scienceqa',
+               split='val',
+               split_file='pid_splits.json',
+               ann_file='problems.json',
+               image_only=True,
+               data_prefix=dict(img_path='val'),
+               pipeline=val_pipeline)
+
+llava_scienceqa_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_scienceqa_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAScienceQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_scienceqa_evaluator = [dict(type='mmpretrain.ScienceQAMetric')]
+
+

configs/multimodal/llava/llava_7b_textvqa.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(
+    type='mmpretrain.TextVQA',
+    data_root='data/textvqa',
+    ann_file='annotations/TextVQA_0.5.1_val.json',
+    pipeline=val_pipeline,
+    data_prefix='images/train_images',
+)
+
+llava_textvqa_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_textvqa_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_textvqa_evaluator = [dict(type='mmpretrain.VQAAcc')]
+
+

configs/multimodal/llava/llava_7b_vizwiz.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(type='mmpretrain.VizWiz',
+               data_root='data/vizwiz/',
+               data_prefix='Images/val',
+               ann_file='Annotations/val.json',
+               pipeline=val_pipeline)
+
+llava_vizwiz_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_vizwiz_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_vizwiz_evaluator = [dict(type='mmpretrain.VQAAcc')]
+
+

configs/multimodal/llava/llava_7b_vqav2.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVABasePostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(
+    type='mmpretrain.COCOVQA',
+    data_root='data/coco',
+    data_prefix='images/val2014',
+    question_file='annotations/v2_OpenEnded_mscoco_val2014_questions.json',
+    ann_file='annotations/v2_mscoco_val2014_annotations.json',
+    pipeline=val_pipeline)
+
+llava_vqav2_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_vqav2_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVABasePostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_vqav2_evaluator = [dict(type='mmpretrain.VQAAcc')]
+
+

configs/multimodal/llava/llava_7b_vsr.py

+from opencompass.multimodal.models.llava import LLaVAVQAPromptConstructor, LLaVAVSRPostProcessor
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(
+        type='mmpretrain.torchvision/Normalize',
+        mean=(0.48145466, 0.4578275, 0.40821073),
+        std=(0.26862954, 0.26130258, 0.27577711),
+    ),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(type='mmpretrain.VSR',
+               data_root='data/vsr/',
+               data_prefix='images/',
+               ann_file='annotations/test.json',
+               pipeline=val_pipeline)
+
+llava_vsr_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    dataset=dataset,
+    collate_fn=dict(type='pseudo_collate'),
+    sampler=dict(type='DefaultSampler', shuffle=False),
+)
+
+# model settings
+llava_vsr_model = dict(
+    type='llava',
+    model_path='/path/to/llava',
+    prompt_constructor=dict(type=LLaVAVQAPromptConstructor),
+    post_processor=dict(type=LLaVAVSRPostProcessor)
+)  # noqa
+
+# evaluation settings
+llava_vsr_evaluator = [dict(type='mmpretrain.GQAAcc')]
+
+

configs/multimodal/visualglm/visualglm_6b_coco_caption.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMBasePromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(type='mmpretrain.PackInputs', algorithm_keys=['image_id'])
+]
+
+
+dataset = dict(type='mmpretrain.COCOCaption',
+               data_root='data/coco',
+               data_prefix=dict(img_path='images'),
+               ann_file='annotations/coco_karpathy_val.json',
+               pipeline=val_pipeline)
+
+visualglm_coco_caption_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_coco_caption_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    is_caption_task=True,
+    prompt_constructor=dict(type=VisualGLMBasePromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_coco_caption_evaluator = [
+    dict(
+        type='mmpretrain.COCOCaption',
+        ann_file='data/coco/annotations/coco_karpathy_val_gt.json',
+    )  # noqa
+]

configs/multimodal/visualglm/visualglm_6b_flickr30k.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMBasePromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(type='mmpretrain.PackInputs', algorithm_keys=['image_id'])
+]
+
+
+dataset = dict(type='mmpretrain.Flickr30kCaption',
+               data_root='data/flickr30k',
+               ann_file='annotations/dataset_flickr30k.json',
+               data_prefix='images',
+               split='val',
+               pipeline=val_pipeline)
+
+visualglm_flickr30k_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_flickr30k_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    is_caption_task=True,
+    prompt_constructor=dict(type=VisualGLMBasePromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_flickr30k_evaluator = [
+    dict(
+        type='mmpretrain.COCOCaption',
+        ann_file='data/flickr30k/annotations/flickr30k_val_gt.json',
+    )  # noqa
+]

configs/multimodal/visualglm/visualglm_6b_gqa.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(type='mmpretrain.GQA',
+               data_root='data/gqa',
+               data_prefix='images',
+               ann_file='annotations/testdev_balanced_questions.json',
+               pipeline=val_pipeline)
+
+visualglm_gqa_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_gqa_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_gqa_evaluator = [dict(type='mmpretrain.GQAAcc')]

configs/multimodal/visualglm/visualglm_6b_mmbench.py

-from opencompass.multimodal.models.visualglm import (VisualGLMPostProcessor, VisualGLMPromptConstructor)
+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMMMBenchPromptConstructor)
 
 # dataloader settings
 val_pipeline = [
@@ -30,8 +30,8 @@ mmbench_dataloader = dict(batch_size=1,
 visualglm_model = dict(
     type='visualglm',
     pretrained_path='/path/to/visualglm',  # or Huggingface repo id
-    prompt_constructor=dict(type=VisualGLMPromptConstructor),
-    post_processor=dict(type=VisualGLMPostProcessor)
+    prompt_constructor=dict(type=VisualGLMMMBenchPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
 )
 
 # evaluation settings

configs/multimodal/visualglm/visualglm_6b_ocr_vqa.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(type='mmpretrain.OCRVQA',
+               data_root='data/ocrvqa',
+               ann_file='annotations/dataset.json',
+               split='test',
+               data_prefix='images',
+               pipeline=val_pipeline)
+
+visualglm_ocrvqa_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_ocrvqa_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_ocrvqa_evaluator = [dict(type='mmpretrain.VQAAcc')]

configs/multimodal/visualglm/visualglm_6b_ok_vqa.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(
+    type='mmpretrain.COCOVQA',
+    data_root='data/okvqa',
+    question_file='annotations/OpenEnded_mscoco_val2014_questions.json',
+    ann_file='annotations/mscoco_val2014_annotations.json',
+    pipeline=val_pipeline,
+    data_prefix='images/val2014',
+)
+
+visualglm_okvqa_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_okvqa_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_okvqa_evaluator = [dict(type='mmpretrain.VQAAcc')]

configs/multimodal/visualglm/visualglm_6b_scienceqa.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMScienceQAPromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(type='mmpretrain.PackInputs',
+         algorithm_keys=[
+             'question', 'gt_answer', 'choices', 'hint', 'lecture', 'solution'
+         ])
+]
+
+dataset = dict(type='mmpretrain.ScienceQA',
+               data_root='./data/scienceqa',
+               split='val',
+               split_file='pid_splits.json',
+               ann_file='problems.json',
+               image_only=True,
+               data_prefix=dict(img_path='val'),
+               pipeline=val_pipeline)
+
+visualglm_vizwiz_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_scienceqa_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    prompt_constructor=dict(type=VisualGLMScienceQAPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_scienceqa_evaluator = [dict(type='mmpretrain.ScienceQAMetric')]
\ No newline at end of file

configs/multimodal/visualglm/visualglm_6b_textvqa.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(
+    type='mmpretrain.TextVQA',
+    data_root='data/textvqa',
+    ann_file='annotations/TextVQA_0.5.1_val.json',
+    pipeline=val_pipeline,
+    data_prefix='images/train_images',
+)
+
+visualglm_textvqa_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_textvqa_evaluator = [dict(type='mmpretrain.VQAAcc')]

configs/multimodal/visualglm/visualglm_6b_vizwiz.py

+from opencompass.multimodal.models.visualglm import (VisualGLMBasePostProcessor, VisualGLMVQAPromptConstructor)
+
+# dataloader settings
+val_pipeline = [
+    dict(type='mmpretrain.LoadImageFromFile'),
+    dict(type='mmpretrain.ToPIL', to_rgb=True),
+    dict(type='mmpretrain.torchvision/Resize',
+         size=(224, 224),
+         interpolation=3),
+    dict(type='mmpretrain.torchvision/ToTensor'),
+    dict(type='mmpretrain.torchvision/Normalize',
+         mean=(0.48145466, 0.4578275, 0.40821073),
+         std=(0.26862954, 0.26130258, 0.27577711)),
+    dict(
+        type='mmpretrain.PackInputs',
+        algorithm_keys=['question', 'gt_answer', 'gt_answer_weight'],
+        meta_keys=['question_id', 'image_id'],
+    )
+]
+
+dataset = dict(type='mmpretrain.VizWiz',
+               data_root='data/vizwiz/',
+               data_prefix='Images/val',
+               ann_file='Annotations/val.json',
+               pipeline=val_pipeline)
+
+visualglm_vizwiz_dataloader = dict(batch_size=1,
+                  num_workers=4,
+                  dataset=dataset,
+                  collate_fn=dict(type='pseudo_collate'),
+                  sampler=dict(type='DefaultSampler', shuffle=False))
+
+# model settings
+visualglm_model = dict(
+    type='visualglm',
+    pretrained_path='/path/to/visualglm',  # or Huggingface repo id
+    prompt_constructor=dict(type=VisualGLMVQAPromptConstructor),
+    post_processor=dict(type=VisualGLMBasePostProcessor)
+)
+
+# evaluation settings
+visualglm_vizwiz_evaluator = [dict(type='mmpretrain.VQAAcc')]