Initial commit

internvl_chat/eval/mmmu/main_eval_only.py

+"""Parse and Evalate"""
+import json
+from argparse import ArgumentParser
+
+from data_utils import CAT_SHORT2LONG, DOMAIN_CAT2SUB_CAT, save_json
+from eval_utils import calculate_ins_level_acc, evaluate, parse_open_response
+
+if __name__ == '__main__':
+
+    parser = ArgumentParser()
+    parser.add_argument('--output_path', type=str, default='./example_outputs/qwen_vl/total_val_output.json',
+                        help='The path to model output file.')
+    parser.add_argument('--answer_path', type=str, default='./answer_dict_val.json', help='Answer file path.')
+    args = parser.parse_args()
+
+    output_dict = json.load(open(args.output_path))
+    answer_dict = json.load(open(args.answer_path))
+
+    # group by category
+    output_dict_w_cat = {}
+    for data_id, parsed_pred in output_dict.items():
+        category = '_'.join(data_id.split('_')[1:-1])
+        if category not in output_dict_w_cat:
+            output_dict_w_cat.update({category: {}})
+        output_dict_w_cat[category].update({data_id: parsed_pred})
+
+    # group by category
+    answer_dict_w_cat = {}
+    for data_id, parsed_pred in answer_dict.items():
+        category = '_'.join(data_id.split('_')[1:-1])
+        if category not in answer_dict_w_cat:
+            answer_dict_w_cat.update({category: {}})
+        answer_dict_w_cat[category].update({data_id: parsed_pred})
+
+    evaluation_result = {}
+
+    for category in CAT_SHORT2LONG.values():
+        print('Evaluating: {}'.format(category))
+        # get cat_outputs and cat_answers
+        try:
+            cat_outputs = output_dict_w_cat[category]
+            cat_answers = answer_dict_w_cat[category]
+        except KeyError:
+            print('Skipping {} for not found'.format(category))
+            continue
+
+        exampels_to_eval = []
+        for data_id, parsed_pred in cat_outputs.items():
+            question_type = cat_answers[data_id]['question_type']
+            if question_type != 'multiple-choice':
+                parsed_pred = parse_open_response(parsed_pred)  # mainly for type consistency (make it number, etc.)
+            else:
+                parsed_pred = parsed_pred
+
+            exampels_to_eval.append({
+                'id': data_id,
+                'question_type': question_type,
+                'answer': cat_answers[data_id]['ground_truth'],
+                'parsed_pred': parsed_pred
+            })
+
+        judge_dict, metric_dict = evaluate(exampels_to_eval)
+        metric_dict.update({'num_example': len(exampels_to_eval)})
+
+        evaluation_result[category] = metric_dict
+
+    printable_results = {}
+    # pdb.set_trace()
+    # add domain Subject
+    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
+        in_domain_cat_results = {}
+        for cat_name in in_domain_cats:  # use the order in DOMAIN_CAT2SUB_CAT
+            if cat_name in evaluation_result.keys():
+                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
+            else:
+                pass
+        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
+        in_domain_data_num = sum([cat_results['num_example'] for cat_results in in_domain_cat_results.values()])
+        printable_results['Overall-' + domain] = {'num': int(in_domain_data_num),
+                                                  'acc': round(in_domain_ins_acc, 3)
+                                                  }
+        # add sub category
+        for cat_name, cat_results in in_domain_cat_results.items():
+            printable_results[cat_name] = {'num': int(cat_results['num_example']),
+                                           'acc': round(cat_results['acc'], 3)
+                                           }
+
+    # table.append(["-----------------------------", "-----", "----"])
+    all_ins_acc = calculate_ins_level_acc(evaluation_result)
+    printable_results['Overall'] = {
+        'num': sum([cat_results['num_example'] for cat_results in evaluation_result.values()]),
+        'acc': round(all_ins_acc, 3)}
+
+    print(printable_results)

internvl_chat/eval/mmmu_pro/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMMU-Pro`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMMU-Pro
+
+The evaluation script will automatically download the MMMU-Pro dataset from HuggingFace, and the cached path is `data/MMMU`.
+
+## 🏃 Evaluation Execution
+
+This evaluation script requires `lmdeploy`. If it's not installed, run the following command:
+
+```shell
+pip install lmdeploy>=0.5.3 --no-deps
+```
+
+To run the evaluation, execute the following command on an 1-GPU setup:
+
+```shell
+python -u eval/mmmu_pro/evaluate_mmmu_pro.py --model ${CHECKPOINT} --mode direct --setting "standard (10 options)" --tp 1
+python -u eval/mmmu_pro/evaluate_mmmu_pro.py --model ${CHECKPOINT} --mode cot --setting "standard (10 options)" --tp 1
+python -u eval/mmmu_pro/evaluate_mmmu_pro.py --model ${CHECKPOINT} --mode direct --setting vision --tp 1
+python -u eval/mmmu_pro/evaluate_mmmu_pro.py --model ${CHECKPOINT} --mode cot --setting vision --tp 1
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=1 sh evaluate.sh ${CHECKPOINT} mmmu-pro-std10 --tp 1
+GPUS=1 sh evaluate.sh ${CHECKPOINT} mmmu-pro-vision --tp 1
+```
+
+After the test is complete, run the following command to get the score:
+
+```shell
+python eval/mmmu_pro/evaluate.py
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument    | Type  | Default                    | Description                                                                                     |
+| ----------- | ----- | -------------------------- | ----------------------------------------------------------------------------------------------- |
+| `--model`   | `str` | `'OpenGVLab/InternVL2-8B'` | Specifies the model name to use in the pipeline.                                                |
+| `--mode`    | `str` | `'direct'`                 | Defines the operation mode, such as `direct` or `cot`.                                          |
+| `--setting` | `str` | `'standard (10 options)'`  | Determines the setting for processing the dataset, such as `standard (10 options)` or `vision`. |
+| `--tp`      | `int` | `1`                        | Sets tensor parallelism (TP) for distributing computations across multiple GPUs.                |

internvl_chat/eval/mmmu_pro/evaluate.py

+import ast
+import json
+import os
+import random
+import re
+from collections import defaultdict
+
+import numpy as np
+
+temp_path = './eval/mmmu_pro/options.jsonl'
+f = open(temp_path, 'r')
+lines = [json.loads(line) for line in f.readlines()]
+id2options = {line['id']: line['options'] for line in lines}
+
+
+def mmmu_process_results(results):
+    pred = results['response']
+    if isinstance(pred, dict):
+        pred = ''
+
+    try:
+        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(str(results['options'])))
+        parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
+    except:
+        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(str(id2options[results['id']])))
+        parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
+
+    id = results['id']
+    if parsed_pred == results['answer']:
+        if_right = True
+    else:
+        if_right = False
+
+    results['pred_indexs'] = parsed_pred
+    results['if_right'] = if_right
+
+    return results
+
+
+def extract_subset_name(input_string):
+    # Define a regex pattern to match "validation_" at the beginning and "_<number>" at the end
+    split = input_string.split('_')[0]
+    pattern = re.compile(rf'^{split}_(.+?)_\d+$')
+    match = pattern.search(input_string)
+    if match:
+        return match.group(1)
+    else:
+        raise ValueError(f'No match found in "{input_string}"')
+
+
+def mmmu_aggregate_results(results):
+    evaluation_result = {}
+    subset_to_eval_samples = defaultdict(list)
+    for result in results:
+        subset_to_eval_samples[result['subdomain']].append(result)
+    for subset, sub_eval_samples in subset_to_eval_samples.items():
+        judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples)
+        metric_dict.update({'num_example': len(sub_eval_samples)})
+        evaluation_result[subset] = metric_dict
+    printable_results = {}
+    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
+        in_domain_cat_results = {}
+        for cat_name in in_domain_cats:
+            if cat_name in evaluation_result.keys():
+                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
+            else:
+                pass
+        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
+        in_domain_data_num = sum([cat_results['num_example'] for cat_results in in_domain_cat_results.values()])
+        printable_results['Overall-' + domain] = {
+            'num': int(in_domain_data_num),
+            'acc': round(in_domain_ins_acc, 3),
+        }
+        # add sub category
+        for cat_name, cat_results in in_domain_cat_results.items():
+            printable_results[cat_name] = {
+                'num': int(cat_results['num_example']),
+                'acc': round(cat_results['acc'], 3),
+            }
+    all_ins_acc = calculate_ins_level_acc(evaluation_result)
+    printable_results['Overall'] = {
+        'num': sum([cat_results['num_example'] for cat_results in evaluation_result.values()]),
+        'acc': round(all_ins_acc, 3),
+    }
+    print(printable_results)
+    return printable_results['Overall']['acc']
+
+
+##################
+# Helper functions written by official MMMU repo.
+##################
+
+
+def calculate_ins_level_acc(results):
+    """Calculate the instruction level accuracy for given Subject results
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246
+    """
+    acc = 0
+    ins_num = 0
+    for cat_results in results.values():
+        acc += cat_results['acc'] * cat_results['num_example']
+        ins_num += cat_results['num_example']
+    if ins_num == 0:
+        return 0
+    return acc / ins_num
+
+
+DOMAIN_CAT2SUB_CAT = {
+    'Art and Design': ['Art', 'Art_Theory', 'Design', 'Music'],
+    'Business': ['Accounting', 'Economics', 'Finance', 'Manage', 'Marketing'],
+    'Science': [
+        'Biology',
+        'Chemistry',
+        'Geography',
+        'Math',
+        'Physics',
+    ],
+    'Health and Medicine': [
+        'Basic_Medical_Science',
+        'Clinical_Medicine',
+        'Diagnostics_and_Laboratory_Medicine',
+        'Pharmacy',
+        'Public_Health',
+    ],
+    'Humanities and Social Science': [
+        'History',
+        'Literature',
+        'Sociology',
+        'Psychology',
+    ],
+    'Tech and Engineering': [
+        'Agriculture',
+        'Architecture_and_Engineering',
+        'Computer_Science',
+        'Electronics',
+        'Energy_and_Power',
+        'Materials',
+        'Mechanical_Engineering',
+    ],
+}
+
+
+def eval_multi_choice(gold_i, pred_i):
+    """
+    Evaluate a multiple choice instance.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175
+    """
+    correct = False
+    # only they are exactly the same, we consider it as correct
+    if isinstance(gold_i, list):
+        for answer in gold_i:
+            if answer == pred_i:
+                correct = True
+                break
+    else:  # gold_i is a string
+        if gold_i == pred_i:
+            correct = True
+    return correct
+
+
+def eval_open(gold_i, pred_i):
+    """
+    Evaluate an open question instance
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191
+    """
+    correct = False
+    if isinstance(gold_i, list):
+        # use float to avoid trivial matches
+        norm_answers = []
+        for answer in gold_i:
+            norm_answers.extend(normalize_str(answer))
+    else:
+        norm_answers = normalize_str(gold_i)
+    for pred in pred_i:  # pred is already normalized in parse response phase
+        if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
+            for norm_ans in norm_answers:
+                # only see if the string answer in the string pred
+                if isinstance(norm_ans, str) and norm_ans in pred:
+                    if not correct:
+                        correct = True
+                    break
+        else:  # it's a float number
+            if pred in norm_answers:
+                if not correct:
+                    correct = True
+                break
+    return correct
+
+
+def evaluate_mmmu(samples):
+    """
+    Batch evaluation for multiple choice and open questions.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219
+    """
+    pred_correct = 0
+    judge_dict = dict()
+    for sample in samples:
+        gold_i = sample['answers']
+        pred_i = sample['parsed_pred']
+
+        correct = eval_multi_choice(gold_i, pred_i)
+
+        if correct:
+            judge_dict[sample['id']] = 'Correct'
+            pred_correct += 1
+        else:
+            judge_dict[sample['id']] = 'Wrong'
+
+    if len(samples) == 0:
+        return {'acc': 0}
+    return judge_dict, {'acc': pred_correct / len(samples)}
+
+
+def parse_multi_choice_responses(response):
+    pred_indexs = []
+    return pred_indexs
+
+
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index, e.g., A, B, C, D.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10
+    """
+    last_answer_pos = response.rfind('Answer:')
+    if last_answer_pos != -1:
+        # Extract the string after "Answer:"
+        answer_str = response[last_answer_pos + len('Answer:'):].strip()
+
+        # Find a unique match in the options
+        matching_options = [option for option in all_choices if option in answer_str]
+
+        # If a unique match is found, return that option
+        if len(matching_options) == 1:
+            return matching_options[0]
+
+    if isinstance(response, str):
+        for char in [',', '.', '!', '?', ';', ':', "'"]:
+            response = response.strip(char)
+        response = ' ' + response + ' '  # add space to avoid partial match
+    else:
+        print(response)
+        response = ''
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f'({choice})' in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A B C D
+            if f'{choice} ' in response:
+                candidates.append(choice)
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A. B. C. D.
+            if f'{choice}.' in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        pred_index = random.choice(all_choices)
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack:
+                for can in candidates:
+                    index = response.rfind(f'({can})')
+                    start_indexes.append(index)  # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f' {can} ')
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:  # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+def extract_numbers(string):
+    """
+    Exact all forms of numbers from a string with regex.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100
+    """
+    # Pattern for numbers with commas
+    pattern_commas = r'-?\b\d{1,3}(?:,\d{3})+\b'
+    # Pattern for scientific notation
+    pattern_scientific = r'-?\d+(?:\.\d+)?[eE][+-]?\d+'
+    # Pattern for simple numbers without commas
+    pattern_simple = r'-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])'
+
+    # Extract numbers with commas
+    numbers_with_commas = re.findall(pattern_commas, string)
+    # Extract numbers in scientific notation
+    numbers_scientific = re.findall(pattern_scientific, string)
+    # Extract simple numbers without commas
+    numbers_simple = re.findall(pattern_simple, string)
+
+    # Combine all extracted numbersz
+    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
+    return all_numbers
+
+
+def check_is_number(string):
+    """
+    Check if the given string a number.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65
+    """
+    try:
+        float(string.replace(',', ''))
+        return True
+    except ValueError:
+        # check if there's comma inside
+        return False
+
+
+def normalize_str(string):
+    """
+    Normalize the str to lower case and make them float numbers if possible.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76
+    """
+    # check if characters in the string
+
+    # if number, numerize it.
+    string = string.strip()
+
+    is_number = check_is_number(string)
+
+    if is_number:
+        string = string.replace(',', '')
+        string = float(string)
+        # leave 2 decimal
+        string = round(string, 2)
+        return [string]
+    else:  # it's likely to be a string
+        # lower it
+        string = string.lower()
+        if len(string) == 1:
+            return [' ' + string, string + ' ']  # avoid trivial matches
+        return [string]
+
+
+def parse_open_response(response):
+    """
+    Parse the prediction from the generated response.
+    Return a list of predicted strings or numbers.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122
+    """
+
+    # content = content.strip("\n").strip(".").strip(" ")
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip().strip('.').lower()
+        sub_responses = re.split(r'\.\s(?=[A-Z])|\n', response)
+        indicators_of_keys = [
+            'could be ',
+            'so ',
+            'is ',
+            'thus ',
+            'therefore ',
+            'final ',
+            'answer ',
+            'result ',
+        ]
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
+            if index == len(sub_responses) - 1:
+                indicators_of_keys.extend(['='])
+            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+                    # key_responses.append(resp.split(indicator)[1].strip())
+
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [
+                    ':',
+                    ',',
+                    '.',
+                    '!',
+                    '?',
+                    ';',
+                    ':',
+                    "'",
+                ]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0:  # did not found any
+            return [response]
+        return key_responses
+
+    # pdb.set_trace()
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy()  # keep the original string response
+    for resp in key_responses:
+        pred_list.extend(extract_numbers(resp))
+
+    tmp_pred_list = []
+    for i in range(len(pred_list)):
+        tmp_pred_list.extend(normalize_str(pred_list[i]))
+    pred_list = tmp_pred_list
+
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+
+def get_multi_choice_info(options):
+    """
+    Given the list of options for multiple choice question
+    Return the index2ans and all_choices
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54
+    """
+
+    start_chr = 'A'
+    all_choices = []
+    index2ans = {}
+    for i, option in enumerate(options):
+        index2ans[chr(ord(start_chr) + i)] = option
+        all_choices.append(chr(ord(start_chr) + i))
+
+    return index2ans, all_choices
+
+
+NUM = 1730
+
+
+def check_files(input_dir):
+    pattern = re.compile(
+        r'(?P<model_name>.+)_(?P<setting>standard \(\d+ options\)|vision)_(?P<method>cot|direct)\.jsonl')
+    for file_name in os.listdir(input_dir):
+        match = pattern.match(file_name)
+        if match:
+            model_name = match.group('model_name')
+            method = match.group('method')
+            setting = match.group('setting')
+            file_path = os.path.join(input_dir, file_name)
+            # print(file_path)
+
+            with open(file_path, 'r', encoding='utf-8') as infile:
+                results = [json.loads(data) for data in infile]
+
+            if len(results) != NUM:
+                print(f'Error: {file_path} has {len(results)} results, expected {NUM}')
+                continue
+
+            processed_results = []
+            true_nums = 0
+            false_nums = 0
+
+            for i, result in enumerate(results):
+                new_data = mmmu_process_results(result)
+                processed_results.append(new_data)
+
+                if new_data['if_right']:
+                    true_nums += 1
+                else:
+                    false_nums += 1
+
+            # Calculate and output the accuracy
+            total = true_nums + false_nums
+            acc = true_nums / total * 100 if total > 0 else 0
+            print(f'Model: {model_name:<15} Method: {method:<8} Setting: {setting:<8} - Accuracy: {acc:>6.2f}%')
+
+            # Write the processed results back to the file
+            with open(file_path, 'w', encoding='utf-8') as outfile:
+                for item in processed_results:
+                    outfile.write(json.dumps(item) + '\n')
+
+
+if __name__ == '__main__':
+    input_directory = './eval/mmmu_pro/results'  # Replace with your input directory
+    check_files(input_directory)

internvl_chat/eval/mmmu_pro/evaluate_mmmu_pro.py

+import argparse
+import ast
+import json
+import os
+
+from datasets import load_dataset
+from lmdeploy import (ChatTemplateConfig, GenerationConfig,
+                      TurbomindEngineConfig, pipeline)
+from tqdm import tqdm
+
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"  # Adjust this if needed
+os.environ['HF_HOME'] = './data/MMMU'
+
+argparse = argparse.ArgumentParser()
+argparse.add_argument('--model', type=str, default='OpenGVLab/InternVL2-8B')
+argparse.add_argument('--mode', type=str, default='direct')
+argparse.add_argument('--setting', type=str, default='standard (10 options)')
+argparse.add_argument('--tp', type=int, default=1)
+args = argparse.parse_args()
+
+MODEL = args.model
+MODE = args.mode
+SETTING = args.setting
+TP = args.tp
+MAX_API_RETRY = 5
+NUM = 1730
+
+import yaml
+
+with open('eval/mmmu_pro/prompts.yaml', 'r') as file:
+    prompt_config = yaml.safe_load(file)[MODE]
+
+
+def replace_images_tokens(input_string):
+    for i in range(1, 8):
+        question_text = f'<image {i}>'
+        query_text = '<image>'
+        if question_text in input_string:
+            input_string = input_string.replace(question_text, query_text)
+    return input_string
+
+
+def parse_options(options):
+    option_letters = [chr(ord('A') + i) for i in range(len(options))]
+    choices_str = '\n'.join([f'{option_letter}. {option}' for option_letter, option in zip(option_letters, options)])
+    return choices_str
+
+
+def construct_prompt(doc):
+    question = doc['question']
+    parsed_options = parse_options(ast.literal_eval(str(doc['options'])))
+    question = f"{question}\n{parsed_options}\n{prompt_config['standard']}"
+    return question
+
+
+def mmmu_doc_to_text(doc):
+    question = construct_prompt(doc)
+    return replace_images_tokens(question)
+
+
+def origin_mmmu_doc_to_visual(doc):
+    visual = []
+    for i in range(1, 8):
+        if not doc[f'image_{i}']:
+            break
+        visual.append(doc[f'image_{i}'])
+    return visual
+
+
+def vision_mmmu_doc_to_visual(doc):
+    return [doc['image']]
+
+
+def process_prompt(data):
+    if 'standard' in SETTING:
+        prompt = mmmu_doc_to_text(data)
+        images = origin_mmmu_doc_to_visual(data)
+    elif SETTING == 'vision':
+        prompt = prompt_config['vision']
+        images = vision_mmmu_doc_to_visual(data)
+    return (prompt, images)
+
+
+def run_and_save(pipe):
+    def save_results_to_file(results, output_path):
+        with open(output_path, 'w', encoding='utf-8') as outfile:
+            for output, data in results:
+                data['response'] = output.text
+                data = {k: v for k, v in data.items() if not k.startswith('image')}
+                json.dump(data, outfile, ensure_ascii=False)
+                outfile.write('\n')
+
+    dataset = load_dataset('MMMU/MMMU_Pro', SETTING, split='test')
+
+    # Process and save dataset parts
+    def process_and_save_part(part_data, part_name, pipe):
+        print(f'Begin processing {part_name}')
+        basename = os.path.basename(MODEL)
+        os.makedirs('./eval/mmmu_pro/results/', exist_ok=True)
+        output_path = f'./eval/mmmu_pro/results/{basename}_{part_name}_{MODE}.jsonl'
+        if os.path.exists(output_path):
+            print(f'Loaded existing results for {part_name}')
+        else:
+            responses = []
+            for data in tqdm(part_data):
+                result = process_prompt(data)
+                response = pipe(result, gen_config=gen_config)
+                print(response)
+                responses.append(response)
+            save_results_to_file(zip(responses, part_data), output_path)
+        return output_path
+
+    gen_config = GenerationConfig(max_new_tokens=4096, temperature=0.0)
+
+    temp_files = []
+    temp_files.append(process_and_save_part(dataset, SETTING, pipe))
+
+
+if __name__ == '__main__':
+    model_name = MODEL.lower().replace('-', '_')
+    if 'internvl2_5' in model_name or 'internvl2.5' in model_name:
+        pipe = pipeline(MODEL, backend_config=TurbomindEngineConfig(session_len=16384, tp=TP),
+                        chat_template_config=ChatTemplateConfig('internvl-internlm2'))
+    elif 'internvl2' in model_name:
+        pipe = pipeline(MODEL, backend_config=TurbomindEngineConfig(session_len=16384, tp=TP))
+    run_and_save(pipe)

internvl_chat/eval/mmmu_pro/prompts.yaml

+cot:
+  vision: "Write out the multiple-choice question in the image and then solve it. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
+  standard: "Answer the preceding multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
+direct:
+  vision: "Answer with the option letter from the given choices directly. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options."
+  standard: "Answer with the option letter from the given choices directly."

internvl_chat/eval/mmvet/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMVet`.
+
+While the provided code can run the benchmark, we recommend using [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) for testing this benchmark if you aim to align results with our technical report.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMVet
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/mm-vet && cd data/mm-vet
+
+# Step 2: Download the dataset
+wget https://github.com/yuweihao/MM-Vet/releases/download/v1/mm-vet.zip
+unzip mm-vet.zip
+wget https://huggingface.co/OpenGVLab/InternVL/raw/main/llava-mm-vet.jsonl
+cd ../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/mm-vet
+ ├── images
+ └── llava-mm-vet.jsonl
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 1-GPU setup:
+
+```shell
+torchrun --nproc_per_node=1 eval/mmvet/evaluate_mmvet.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=1 sh evaluate.sh ${CHECKPOINT} mmvet --dynamic
+```
+
+After the test is completed, a file with a name similar to `results/mmvet_241224214015.json` will be generated. Please upload this file to the [official server](https://huggingface.co/spaces/whyu/MM-Vet_Evaluator) to obtain the evaluation scores.
+
+> ⚠️ Note: The test scores from the official server of MMVet will be significantly higher than those of VLMEvalKit. To align the scores with our technical report, please use VLMEvalKit to test this benchmark.
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default   | Description                                                                                                       |
+| ---------------- | ------ | --------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`      | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'mmvet'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`   | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`       | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`   | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`   | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mmvet/evaluate_mmvet.py

+import argparse
+import json
+import os
+import random
+import time
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'mmvet': {
+        'root': 'data/mm-vet/images',
+        'question': 'data/mm-vet/llava-mm-vet.jsonl',
+        'metric': None,
+        'max_new_tokens': 1000,
+        'min_new_tokens': 1,
+    }
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    question_ids = [_['question_id'] for _ in batches]
+    annotations = [_['annotation'] for _ in batches]
+
+    return pixel_values, questions, question_ids, annotations
+
+
+class VQADataset(torch.utils.data.Dataset):
+
+    def __init__(self, root, data, prompt, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6):
+        self.root = root
+        self.data = open(data).readlines()
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        data = json.loads(self.data[idx].strip())
+        image, question, question_id, annotation = data['image'], data[
+            'text'], data['question_id'], data.get('answer', None)
+
+        image = os.path.join(self.root, image)
+        image = Image.open(image).convert('RGB')
+        if self.dynamic_image_size:
+            images = dynamic_preprocess(image, image_size=self.input_size,
+                                        use_thumbnail=self.use_thumbnail,
+                                        max_num=self.max_num)
+        else:
+            images = [image]
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+        question = question + ' ' + self.prompt
+        return question_id, question, pixel_values, annotation
+
+
+def evaluate_chat_model():
+    random.seed(args.seed)
+    prompt = ''
+
+    for ds_name in args.datasets:
+        dataset = VQADataset(
+            root=ds_collections[ds_name]['root'],
+            data=ds_collections[ds_name]['question'],
+            prompt=prompt,
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+
+        outputs = {}
+        for _, (question_id, question, pixel_values, annotations) in tqdm(enumerate(dataset)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            pred = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=question,
+                generation_config=generation_config,
+                verbose=True
+            )
+
+            outputs[f'v1_{question_id}'] = pred
+
+        print(f'Evaluating {ds_name} ...')
+        time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+        results_file = f'{ds_name}_{time_prefix}.json'
+        results_file = os.path.join(args.out_dir, results_file)
+        json.dump(outputs, open(results_file, 'w'))
+        print('Results saved to {}'.format(results_file))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='mmvet')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+
+    evaluate_chat_model()

internvl_chat/eval/mmvetv2/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMVet v2`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMVet v2
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/mm-vet-v2 && cd data/mm-vet-v2
+
+# Step 2: Download the dataset
+wget https://github.com/yuweihao/MM-Vet/releases/download/v2/mm-vet-v2.zip
+unzip mm-vet-v2.zip
+
+cd ../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/mm-vet-v2
+ ├── images
+ └── mm-vet-v2.json
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/mmvetv2/evaluate_mmvet_v2.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} mmvetv2 --dynamic
+```
+
+After the test is completed, a file with a name similar to `results/mmvet-v2_241224214015.json` will be generated. Please upload this file to the [official server](https://huggingface.co/spaces/whyu/MM-Vet-v2_Evaluator) to obtain the evaluation scores.
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default      | Description                                                                                                       |
+| ---------------- | ------ | ------------ | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`         | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'mmvet-v2'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`      | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`          | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`      | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`      | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mmvetv2/evaluate_mmvet_v2.py

+import argparse
+import itertools
+import json
+import os
+import random
+import time
+from functools import partial
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'mmvet-v2': {
+        'root': 'data/mm-vet-v2/images',
+        'question': 'data/mm-vet-v2/mm-vet-v2.json',
+        'metric': None,
+        'max_new_tokens': 1000,
+        'min_new_tokens': 1,
+    }
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    question_ids = [_['question_id'] for _ in batches]
+    annotations = [_['annotation'] for _ in batches]
+
+    return pixel_values, questions, question_ids, annotations
+
+
+class VQADataset(torch.utils.data.Dataset):
+
+    def __init__(self, root, data, prompt, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6):
+        self.root = root
+        with open(data, 'r') as f:
+            data = json.loads(f.read())
+            self.data = [(k, v) for k, v in data.items()]
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        data = self.data[idx]
+        question_id = data[0]
+        question = data[1]['question'].split('<IMG>')[0]
+        question = question + ' ' + self.prompt
+        image = data[1]['question'].split('<IMG>')[1]
+        annotation = data[1]['answer']
+        added_in = data[1]['added_in']
+
+        image = os.path.join(self.root, image)
+        image = Image.open(image).convert('RGB')
+        if self.dynamic_image_size:
+            images = dynamic_preprocess(image, image_size=self.input_size,
+                                        use_thumbnail=self.use_thumbnail,
+                                        max_num=self.max_num)
+        else:
+            images = [image]
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+        return {
+            'pixel_values': pixel_values,
+            'question': question,
+            'question_id': question_id,
+            'annotation': annotation,
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def evaluate_chat_model():
+    random.seed(args.seed)
+
+    for ds_name in args.datasets:
+        dataset = VQADataset(
+            root=ds_collections[ds_name]['root'],
+            data=ds_collections[ds_name]['question'],
+            prompt='Answer this question in detail.',
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, question_ids, annotations) in tqdm(enumerate(dataloader)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            pred = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=questions[0],
+                generation_config=generation_config,
+                verbose=True
+            )
+            outputs.append({
+                'question_id': question_ids[0],
+                'prediction': pred
+            })
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.json'
+            results_file = os.path.join(args.out_dir, results_file)
+            outputs = {}
+            for item in merged_outputs:
+                question_id = item['question_id']
+                prediction = item['prediction']
+                outputs[question_id] = prediction
+            json.dump(outputs, open(results_file, 'w'))
+            print('Results saved to {}'.format(results_file))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='mmvet-v2')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()

internvl_chat/eval/mmvp/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMVP`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMVP
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Download the dataset
+cd data/
+git clone https://huggingface.co/datasets/MMVP/MMVP
+cd ../
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/MMVP
+ ├── MMVP Images
+ ├── Questions.csv
+ ├── Questions.xlsx
+ └── README.md
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/mmvp/evaluate_mmvp.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} mmvp --dynamic
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default  | Description                                                                                                       |
+| ---------------- | ------ | -------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`     | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'MMVP'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`  | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`      | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`  | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`  | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mmvp/evaluate_mmvp.py

+import argparse
+import csv
+import itertools
+import json
+import os
+import random
+import time
+from functools import partial
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'MMVP': {
+        'root': 'data/MMVP',
+        'max_new_tokens': 100,
+        'min_new_tokens': 1,
+    },
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    answers = [_['answer'] for _ in batches]
+    data_ids = [_['data_id'] for _ in batches]
+    options = [_['option'] for _ in batches]
+    return pixel_values, questions, answers, data_ids, options
+
+
+class MMVPDataset(torch.utils.data.Dataset):
+
+    def __init__(self, root, prompt, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6):
+        # run for each subject
+        meta_path = os.path.join(root, 'Questions.csv')
+        with open(meta_path, 'r') as file:
+            csv_reader = csv.DictReader(file)
+            data = [row for row in csv_reader]
+
+        self.data = data
+        self.root = root
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        data = self.data[idx]
+        data_id = data['lndex'] if 'lndex' in data else data['Index']
+        question = data['Question']
+        image = os.path.join(self.root + '/MMVP Images', data_id + '.jpg')
+        image = Image.open(image).convert('RGB')
+
+        options = data['Options'].split('(b)')
+        options[0] = options[0].replace('(a)', '').strip()
+        options[1] = options[1].replace('(b)', '').strip()
+
+        answer = data['Correct Answer'] if 'Correct Answer' in data else None
+        answer = answer.replace('(a)', 'A').replace('(b)', 'B').replace('(c)', 'C').replace('(d)', 'D')
+        choice_list = []
+        new_options = {}
+        multiple_choices = ['A', 'B', 'C', 'D']
+        for i, c in enumerate(options):
+            choice_list.append('{}. {}'.format(multiple_choices[i], c.strip()))
+            new_options[multiple_choices[i]] = c.strip()
+        choice_txt = '\n'.join(choice_list)
+
+        if self.dynamic_image_size:
+            images = dynamic_preprocess(image, image_size=self.input_size,
+                                        use_thumbnail=self.use_thumbnail,
+                                        max_num=self.max_num)
+        else:
+            images = [image]
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+
+        if len(choice_txt) > 0:
+            question += '\n' + choice_txt
+        question += '\n' + self.prompt
+        return {
+            'question': question,
+            'pixel_values': pixel_values,
+            'answer': answer,
+            'option': new_options,
+            'data_id': data_id
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def post_process(pred, option):
+    pred = pred.strip()
+    option_candidate = list(option.keys())
+    if len(pred) == 1:
+        return pred
+    elif len(pred) != 1 and pred[0] in option_candidate:
+        return pred[0]
+    elif len(pred) != 1 and pred[0] not in option_candidate:
+        for k, v in option.items():
+            if v in pred:
+                return k
+
+    return pred
+
+
+def evaluate_chat_model():
+    prompt = "Answer with the option's letter from the given choices directly."
+    random.seed(args.seed)
+
+    for ds_name in args.datasets:
+        dataset = MMVPDataset(
+            root=ds_collections[ds_name]['root'],  # hf dataset path.
+            prompt=prompt,
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, answers, data_ids, options) in tqdm(enumerate(dataloader)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            pred = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=questions[0],
+                generation_config=generation_config,
+                verbose=True
+            )
+            if len(options[0]) == 0:
+                preds = [pred]
+            else:
+                preds = [post_process(pred, options[0])]
+
+            for question, pred, answer, data_id in zip(questions, preds, answers, data_ids):
+                outputs.append({
+                    'question': question,
+                    'answer': pred,
+                    'gt_answers': answer,
+                    'data_id': data_id
+                })
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.jsonl'
+            output_path = os.path.join(args.out_dir, results_file)
+            writer = open(output_path, 'w')
+
+            num_correct, num_total = 0, 0
+            index, round_correct = 0, 0
+            for item in merged_outputs:
+                writer.write(json.dumps(item) + '\n')
+                answer = item['answer']
+                gt_answer = item['gt_answers']
+
+                index += 1
+
+                if answer == gt_answer:
+                    round_correct += 1
+                if index == 2:
+                    index = 0
+                    if round_correct == 2:
+                        num_correct += 1
+                    round_correct = 0
+
+                    num_total += 1
+
+            writer.close()
+            print('Results saved to {}'.format(output_path))
+            print(f'The accuracy is {num_correct/num_total}')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='MMVP')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()

internvl_chat/eval/mpdocvqa/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MP-DocVQA`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MP-DocVQA
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/mpdocvqa && cd data/mpdocvqa
+
+# Step 2: Download the dataset
+# Download from https://rrc.cvc.uab.es/?ch=17&com=downloads
+
+cd ../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/mpdocvqa
+ ├── images
+ ├── test.json
+ ├── train.json
+ └── val.json
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+# Test the val set
+torchrun --nproc_per_node=8 eval/mpdocvqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets mpdocvqa_val --dynamic --max-num 18
+# Test the test set
+torchrun --nproc_per_node=8 eval/mpdocvqa/evaluate_vqa.py --checkpoint ${CHECKPOINT} --datasets mpdocvqa_val --dynamic --max-num 18
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+# Test the val set
+GPUS=8 sh evaluate.sh ${CHECKPOINT} vqa-mpdocvqa-val --dynamic --max-num 18
+# Test the test set
+GPUS=8 sh evaluate.sh ${CHECKPOINT} vqa-mpdocvqa-test --dynamic --max-num 18
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default          | Description                                                                                                       |
+| ---------------- | ------ | ---------------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`             | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'mpdocvqa_val'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`          | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `18`             | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`          | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`          | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mpdocvqa/evaluate_vqa.py

+import argparse
+import itertools
+import json
+import os
+import random
+import time
+from functools import partial
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'mpdocvqa_val': {
+        'root': 'data/mpdocvqa/images/',
+        'test': 'data/mpdocvqa/val.json',
+        'annotation': 'data/mpdocvqa/val.json',
+        'metric': 'anls',
+        'max_new_tokens': 100,
+    },
+    'mpdocvqa_test': {
+        'root': 'data/mpdocvqa/images/',
+        'test': 'data/mpdocvqa/test.json',
+        'metric': None,
+        'max_new_tokens': 100,
+    }
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    question_ids = [_['question_id'] for _ in batches]
+    annotations = [_['annotation'] for _ in batches]
+    num_patches_lists = [_['num_patches_list'] for _ in batches]
+
+    return pixel_values, questions, question_ids, annotations, num_patches_lists
+
+
+class VQADataset(torch.utils.data.Dataset):
+
+    def __init__(self, root, test, prompt, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6, total_max_num=64):
+        self.root = root
+        self.test = json.loads(open(test).read())['data']
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.total_max_num = total_max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.test)
+
+    def __getitem__(self, idx):
+        data = self.test[idx]
+        page_ids = data['page_ids']
+        question_id = data['questionId']
+        question = data['question']
+        annotation = data.get('answers', None)
+        image_list = []
+        for page_id in page_ids:
+            image_path = os.path.join(self.root, page_id + '.jpg')
+            image = Image.open(image_path).convert('RGB')
+            image_list.append(image)
+
+        max_num = max(1, min(self.max_num, self.total_max_num // len(image_list)))
+        num_patches_list = []
+        if self.dynamic_image_size:
+            images = []
+            for image in image_list:
+                tiles = dynamic_preprocess(image, image_size=self.input_size,
+                                           use_thumbnail=self.use_thumbnail,
+                                           max_num=max_num)
+                images += tiles
+                num_patches_list.append(len(tiles))
+        else:
+            images = image_list
+            num_patches_list.append(1)
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+
+        if len(images) > 1:
+            prefix = ''.join([f'Image-{i + 1}: <image>\n' for i in range(len(image_list))])
+            question = prefix + question
+        if len(self.prompt) != 0:
+            question = question + ' ' + self.prompt
+        return {
+            'question_id': question_id,
+            'question': question,
+            'pixel_values': pixel_values,
+            'annotation': annotation,
+            'num_patches_list': num_patches_list
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def evaluate_chat_model():
+    base_prompt = 'Answer the question using a single word or phrase.'
+    random.seed(args.seed)
+    summaries = []
+
+    for ds_name in args.datasets:
+        dataset = VQADataset(
+            root=ds_collections[ds_name]['root'],
+            test=ds_collections[ds_name]['test'],
+            prompt=base_prompt,
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num,
+            total_max_num=args.total_max_num,
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, question_ids, annotations, num_patches_lists) in tqdm(enumerate(dataloader)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=1,
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            with torch.inference_mode():
+                pred = model.chat(
+                    tokenizer=tokenizer,
+                    pixel_values=pixel_values,
+                    question=questions[0],
+                    generation_config=generation_config,
+                    num_patches_list=num_patches_lists[0],
+                    verbose=True
+                )
+                torch.cuda.empty_cache()
+            answers = [pred]
+
+            for question, question_id, answer, annotation in zip(questions, question_ids, answers, annotations):
+                if ds_name in ['mpdocvqa_val']:
+                    outputs.append({
+                        'question': question,
+                        'questionId': question_id,
+                        'answer': answer,
+                        'annotation': annotation,
+                    })
+                elif ds_name in ['mpdocvqa_test']:
+                    outputs.append({
+                        'questionId': question_id,
+                        'answer': answer,
+                    })
+                else:
+                    raise NotImplementedError
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.json'
+            results_file = os.path.join(args.out_dir, results_file)
+            json.dump(merged_outputs, open(results_file, 'w'))
+            print('Results saved to {}'.format(results_file))
+
+            if ds_collections[ds_name]['metric'] == 'anls':
+                json.dump(merged_outputs,
+                          open(results_file, 'w'),
+                          ensure_ascii=False)
+                print('python eval/mpdocvqa/infographicsvqa_eval.py -g ' +
+                      ds_collections[ds_name]['annotation'] + ' -s ' +
+                      results_file)
+                os.system('python eval/mpdocvqa/infographicsvqa_eval.py -g ' +
+                          ds_collections[ds_name]['annotation'] + ' -s ' +
+                          results_file)
+
+        torch.distributed.barrier()
+
+    out_path = '_'.join(args.checkpoint.split('/')[-2:])
+    writer = open(os.path.join(args.out_dir, f'{out_path}.txt'), 'a')
+    print(f"write results to file {os.path.join(args.out_dir, f'{out_path}.txt')}")
+    for summary in summaries:
+        print(summary)
+        writer.write(f'{summary}\n')
+    writer.close()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='mpdocvqa_val')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=18)
+    parser.add_argument('--total-max-num', type=int, default=64)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()

internvl_chat/eval/mpdocvqa/infographicsvqa_eval.py

+# This file can be downloaded from: https://www.docvqa.org/datasets/infographicvqa and https://rrc.cvc.uab.es/?ch=17&com=introduction
+
+import argparse
+import json
+import os
+
+question_ids_to_exclude = []
+
+# answer_types = {'image span': 'Image-Span', 'question span': 'Question-Span', 'multiple spans': 'Multi-Span', 'non span': 'None span', 'list': 'List'}
+answer_types = {'image span': 'Image-Span', 'question span': 'Question-Span', 'multiple spans': 'Multi-Span',
+                'non span': 'None span'}
+evidence_types = {'table/list': 'Table/list', 'textual': 'Text', 'photo/pciture/visual_objects': 'Visual/Layout',
+                  'figure': 'Figure', 'map': 'Map'}
+reasoning_requirements = {'comparison': 'Sorting', 'arithmetic': 'Arithmetic', 'counting': 'Counting'}
+
+
+def save_json(file_path, data):
+    with open(file_path, 'w+') as json_file:
+        json.dump(data, json_file)
+
+
+def levenshtein_distance(s1, s2):
+    if len(s1) > len(s2):
+        s1, s2 = s2, s1
+
+    distances = range(len(s1) + 1)
+    for i2, c2 in enumerate(s2):
+        distances_ = [i2 + 1]
+        for i1, c1 in enumerate(s1):
+            if c1 == c2:
+                distances_.append(distances[i1])
+            else:
+                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
+        distances = distances_
+    return distances[-1]
+
+
+def validate_data(gtFilePath, submFilePath):
+    """
+    Method validate_data: validates that all files in the results folder are correct (have the correct name contents).
+                            Validates also that there are no missing files in the folder.
+                            If some error detected, the method raises the error
+    """
+
+    gtJson = json.load(open(gtFilePath, 'rb'))
+    submJson = json.load(open(submFilePath, 'rb'))
+
+    if 'data' not in gtJson:
+        raise Exception('The GT file is not valid (no data key)')
+
+    if 'dataset_name' not in gtJson:
+        raise Exception('The GT file is not valid (no dataset_name key)')
+
+    if isinstance(submJson, list) is False:
+        raise Exception('The Det file is not valid (root item must be an array)')
+
+    if len(submJson) != len(gtJson['data']):
+        raise Exception('The Det file is not valid (invalid number of answers. Expected:' + str(
+            len(gtJson['data'])) + ' Found:' + str(len(submJson)) + ')')
+
+    gtQuestions = sorted([r['questionId'] for r in gtJson['data']])
+    res_id_to_index = {int(r['questionId']): ix for ix, r in enumerate(submJson)}
+    detQuestions = sorted([r['questionId'] for r in submJson])
+
+    if ((gtQuestions == detQuestions) is False):
+        raise Exception('The Det file is not valid. Question IDs must much GT')
+
+    for gtObject in gtJson['data']:
+
+        try:
+            q_id = int(gtObject['questionId'])
+            res_ix = res_id_to_index[q_id]
+
+        except:
+            raise Exception('The Det file is not valid. Question ' + str(gtObject['questionId']) + ' not present')
+
+        else:
+            detObject = submJson[res_ix]
+
+            #            if detObject['questionId'] != gtObject['questionId'] :
+            #                raise Exception("Answer #" + str(i) + " not valid (invalid question ID. Expected:" + str(gtObject['questionId']) + "Found:" + detObject['questionId'] + ")")
+
+            if 'answer' not in detObject:
+                raise Exception('Question ' + str(gtObject['questionId']) + ' not valid (no answer key)')
+
+            if isinstance(detObject['answer'], list) is True:
+                raise Exception(
+                    'Question ' + str(gtObject['questionId']) + ' not valid (answer key has to be a single string)')
+
+
+def evaluate_method(gtFilePath, submFilePath, evaluationParams):
+    """
+    Method evaluate_method: evaluate method and returns the results
+        Results. Dictionary with the following values:
+        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
+        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
+    """
+
+    show_scores_per_answer_type = evaluationParams.answer_types
+
+    gtJson = json.load(open(gtFilePath, 'rb'))
+    submJson = json.load(open(submFilePath, 'rb'))
+
+    res_id_to_index = {int(r['questionId']): ix for ix, r in enumerate(submJson)}
+
+    perSampleMetrics = {}
+
+    totalScore = 0
+    row = 0
+
+    if show_scores_per_answer_type:
+        answerTypeTotalScore = {x: 0 for x in answer_types.keys()}
+        answerTypeNumQuestions = {x: 0 for x in answer_types.keys()}
+
+        evidenceTypeTotalScore = {x: 0 for x in evidence_types.keys()}
+        evidenceTypeNumQuestions = {x: 0 for x in evidence_types.keys()}
+
+        reasoningTypeTotalScore = {x: 0 for x in reasoning_requirements.keys()}
+        reasoningTypeNumQuestions = {x: 0 for x in reasoning_requirements.keys()}
+
+    for gtObject in gtJson['data']:
+
+        q_id = int(gtObject['questionId'])
+        res_ix = res_id_to_index[q_id]
+        detObject = submJson[res_ix]
+
+        if q_id in question_ids_to_exclude:
+            question_result = 0
+            info = 'Question EXCLUDED from the result'
+
+        else:
+            info = ''
+            values = []
+            for answer in gtObject['answers']:
+                # preprocess both the answers - gt and prediction
+                gt_answer = ' '.join(answer.strip().lower().split())
+                det_answer = ' '.join(detObject['answer'].strip().lower().split())
+
+                # dist = levenshtein_distance(answer.lower(), detObject['answer'].lower())
+                dist = levenshtein_distance(gt_answer, det_answer)
+                length = max(len(answer.upper()), len(detObject['answer'].upper()))
+                values.append(0.0 if length == 0 else float(dist) / float(length))
+
+            question_result = 1 - min(values)
+
+            if (question_result < evaluationParams.anls_threshold):
+                question_result = 0
+
+            totalScore += question_result
+
+            if show_scores_per_answer_type:
+                for q_type in gtObject['answer_type']:
+                    answerTypeTotalScore[q_type] += question_result
+                    answerTypeNumQuestions[q_type] += 1
+
+                for q_type in gtObject['evidence']:
+                    evidenceTypeTotalScore[q_type] += question_result
+                    evidenceTypeNumQuestions[q_type] += 1
+
+                for q_type in gtObject['operation/reasoning']:
+                    reasoningTypeTotalScore[q_type] += question_result
+                    reasoningTypeNumQuestions[q_type] += 1
+
+        perSampleMetrics[str(gtObject['questionId'])] = {
+            'score': question_result,
+            'question': gtObject['question'],
+            'gt': gtObject['answers'],
+            'det': detObject['answer'],
+            'info': info
+        }
+        row = row + 1
+
+    methodMetrics = {
+        'score': 0 if len(gtJson['data']) == 0 else totalScore / (len(gtJson['data']) - len(question_ids_to_exclude))
+    }
+
+    answer_types_scores = {}
+    evidence_types_scores = {}
+    operation_types_scores = {}
+
+    if show_scores_per_answer_type:
+        for a_type, ref in answer_types.items():
+            answer_types_scores[ref] = 0 if len(gtJson['data']) == 0 else answerTypeTotalScore[a_type] / (
+            answerTypeNumQuestions[a_type])
+
+        for e_type, ref in evidence_types.items():
+            evidence_types_scores[ref] = 0 if len(gtJson['data']) == 0 else evidenceTypeTotalScore[e_type] / (
+            evidenceTypeNumQuestions[e_type])
+
+        for r_type, ref in reasoning_requirements.items():
+            operation_types_scores[ref] = 0 if len(gtJson['data']) == 0 else reasoningTypeTotalScore[r_type] / (
+            reasoningTypeNumQuestions[r_type])
+
+    resDict = {
+        'result': methodMetrics,
+        'scores_by_types': {'answer_types': answer_types_scores, 'evidence_types': evidence_types_scores,
+                            'operation_types': operation_types_scores},
+        'per_sample_result': perSampleMetrics
+    }
+
+    return resDict
+
+
+def display_results(results, show_answer_types):
+    print('\nOverall ANLS: {:2.4f}'.format(results['result']['score']))
+
+    if show_answer_types:
+        print('\nAnswer types:')
+        for a_type in answer_types.values():
+            print('\t{:12s} {:2.4f}'.format(a_type, results['scores_by_types']['answer_types'][a_type]))
+
+        print('\nEvidence types:')
+        for e_type in evidence_types.values():
+            print('\t{:12s} {:2.4f}'.format(e_type, results['scores_by_types']['evidence_types'][e_type]))
+
+        print('\nOperation required:')
+        for r_type in reasoning_requirements.values():
+            print('\t{:12s} {:2.4f}'.format(r_type, results['scores_by_types']['operation_types'][r_type]))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='InfographVQA evaluation script.')
+
+    parser.add_argument('-g', '--ground_truth', type=str, help='Path of the Ground Truth file.', required=True)
+    parser.add_argument('-s', '--submission_file', type=str, help="Path of your method's results file.", required=True)
+
+    parser.add_argument('-t', '--anls_threshold', type=float, default=0.5,
+                        help='ANLS threshold to use (See Scene-Text VQA paper for more info.).', required=False)
+    parser.add_argument('-a', '--answer_types', type=bool, default=False,
+                        help='Score break down by answer types (special gt file required).', required=False)
+    parser.add_argument('-o', '--output', type=str,
+                        help="Path to a directory where to copy the file 'results.json' that contains per-sample results.",
+                        required=False)
+
+    args = parser.parse_args()
+
+    # Validate the format of ground truth and submission files.
+    validate_data(args.ground_truth, args.submission_file)
+
+    # Evaluate method
+    results = evaluate_method(args.ground_truth, args.submission_file, args)
+
+    display_results(results, args.answer_types)
+
+    if args.output:
+        output_dir = args.output
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        resultsOutputname = os.path.join(output_dir, 'results.json')
+        save_json(resultsOutputname, results)
+
+        print('All results including per-sample result has been correctly saved!')

internvl_chat/eval/mvbench/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MVBench`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MVBench
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Download the dataset
+cd data/
+huggingface-cli download --repo-type dataset --resume-download OpenGVLab/MVBench --local-dir MVBench --local-dir-use-symlinks False
+
+# Step 2: Unzip videos
+cd MVBench/video/
+for file in *.zip; do unzip "$file" -d "${file%.*}"; done
+cd ../../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/MVBench
+├── json
+│   ├── action_antonym.json
+│   ├── action_count.json
+│   ├── action_localization.json
+│   ├── action_prediction.json
+│   ├── action_sequence.json
+│   ├── character_order.json
+│   ├── counterfactual_inference.json
+│   ├── egocentric_navigation.json
+│   ├── episodic_reasoning.json
+│   ├── fine_grained_action.json
+│   ├── fine_grained_pose.json
+│   ├── moving_attribute.json
+│   ├── moving_count.json
+│   ├── moving_direction.json
+│   ├── object_existence.json
+│   ├── object_interaction.json
+│   ├── object_shuffle.json
+│   ├── scene_transition.json
+│   ├── state_change.json
+│   └── unexpected_action.json
+├── README.md
+└── video
+    ├── clevrer
+    ├── FunQA_test
+    ├── Moments_in_Time_Raw
+    ├── nturgbd
+    ├── perception
+    ├── scene_qa
+    ├── ssv2_video
+    ├── sta
+    ├── star
+    ├── tvqa
+    └── vlnqa
+```
+
+## 🏃 Evaluation Execution
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/mvbench/evaluate_mvbench.py --checkpoint ${CHECKPOINT} --num_segments 16
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} mvbench
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default     | Description                                                                                                       |
+| ---------------- | ------ | ----------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`        | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'mvbench'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`     | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `1`         | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`     | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`     | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mvbench/evaluate_mvbench.py

+import argparse
+import itertools
+import json
+import os
+import random
+import time
+from functools import partial
+
+import cv2
+import imageio
+import numpy as np
+import torch
+from decord import VideoReader, cpu
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+data_list = {
+    'Action Sequence': ('action_sequence.json', './data/MVBench/video/star/Charades_v1_480/', 'video', True),
+    # has start & end
+    'Action Prediction': ('action_prediction.json', './data/MVBench/video/star/Charades_v1_480/', 'video', True),
+    # has start & end
+    'Action Antonym': ('action_antonym.json', './data/MVBench/video/ssv2_video/', 'video', False),
+    'Fine-grained Action': (
+    'fine_grained_action.json', './data/MVBench/video/Moments_in_Time_Raw/videos/', 'video', False),
+    'Unexpected Action': ('unexpected_action.json', './data/MVBench/video/FunQA_test/test/', 'video', False),
+    'Object Existence': ('object_existence.json', './data/MVBench/video/clevrer/video_validation/', 'video', False),
+    'Object Interaction': ('object_interaction.json', './data/MVBench/video/star/Charades_v1_480/', 'video', True),
+    # has start & end
+    'Object Shuffle': ('object_shuffle.json', './data/MVBench/video/perception/videos/', 'video', False),
+    'Moving Direction': ('moving_direction.json', './data/MVBench/video/clevrer/video_validation/', 'video', False),
+    'Action Localization': ('action_localization.json', './data/MVBench/video/sta/sta_video/', 'video', True),
+    # has start & end
+    'Scene Transition': ('scene_transition.json', './data/MVBench/video/scene_qa/video/', 'video', False),
+    'Action Count': ('action_count.json', './data/MVBench/video/perception/videos/', 'video', False),
+    'Moving Count': ('moving_count.json', './data/MVBench/video/clevrer/video_validation/', 'video', False),
+    'Moving Attribute': ('moving_attribute.json', './data/MVBench/video/clevrer/video_validation/', 'video', False),
+    'State Change': ('state_change.json', './data/MVBench/video/perception/videos/', 'video', False),
+    'Fine-grained Pose': ('fine_grained_pose.json', './data/MVBench/video/nturgbd/', 'video', False),
+    'Character Order': ('character_order.json', './data/MVBench/video/perception/videos/', 'video', False),
+    'Egocentric Navigation': ('egocentric_navigation.json', './data/MVBench/video/vlnqa/', 'video', False),
+    'Episodic Reasoning': ('episodic_reasoning.json', './data/MVBench/video/tvqa/frames_fps3_hq/', 'frame', True),
+    # has start & end, read frame
+    'Counterfactual Inference': (
+    'counterfactual_inference.json', './data/MVBench/video/clevrer/video_validation/', 'video', False),
+}
+
+data_dir = './data/MVBench/json'
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    answers = [_['answer'] for _ in batches]
+    num_patches_lists = [_['num_patches_list'] for _ in batches]
+    task_types = [_['task_type'] for _ in batches]
+    return pixel_values, questions, answers, num_patches_lists, task_types
+
+
+class MVBenchDataset(torch.utils.data.Dataset):
+
+    def __init__(self, data_dir, data_list, prompt, question_prompt, num_segments=16, input_size=224,
+                 dynamic_image_size=False, use_thumbnail=False, max_num=6):
+        self.data_list = []
+        for k, v in data_list.items():
+            with open(os.path.join(data_dir, v[0]), 'r') as f:
+                json_data = json.load(f)
+            for data in json_data:
+                self.data_list.append({
+                    'task_type': k,
+                    'prefix': v[1],
+                    'data_type': v[2],
+                    'bound': v[3],
+                    'data': data
+                })
+        self.decord_method = {
+            'video': self.read_video,
+            'gif': self.read_gif,
+            'frame': self.read_frame,
+        }
+        self.prompt = prompt
+        self.question_prompt = question_prompt
+        self.input_size = input_size
+        self.num_segments = num_segments
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data_list)
+
+    def __str__(self):
+        len_list = {}
+        option_list = {}
+        for data in self.data_list:
+            if data['task_type'] not in len_list:
+                len_list[data['task_type']] = 0
+            len_list[data['task_type']] += 1
+            if data['task_type'] not in option_list:
+                option_list[data['task_type']] = 0
+            option_list[data['task_type']] += len(data['data']['candidates'])
+
+        correct = 0
+        total = 0
+        res = f'There are {len(self.data_list)} videos as follow:\n'
+        for k, v in len_list.items():
+            correct += len_list[k]
+            total += option_list[k]
+            res += f'{v} for {k} ({option_list[k]} options => {len_list[k] / option_list[k] * 100:.2f}%)\n'
+            correct = correct + 1 / option_list[k]
+        res += f'Total random accuracy: {correct / total * 100:.2f}%'
+        return res.rstrip()
+
+    def get_index(self, bound, fps, max_frame, first_idx=0):
+        if bound:
+            start, end = bound[0], bound[1]
+        else:
+            start, end = -100000, 100000
+        start_idx = max(first_idx, round(start * fps))
+        end_idx = min(round(end * fps), max_frame)
+        seg_size = float(end_idx - start_idx) / self.num_segments
+        frame_indices = np.array([
+            int(start_idx + (seg_size / 2) + np.round(seg_size * idx))
+            for idx in range(self.num_segments)
+        ])
+        return frame_indices
+
+    def read_video(self, video_path, bound=None):
+        vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
+        max_frame = len(vr) - 1
+        fps = float(vr.get_avg_fps())
+
+        images_group = list()
+        frame_indices = self.get_index(bound, fps, max_frame, first_idx=0)
+        for frame_index in frame_indices:
+            img = Image.fromarray(vr[frame_index].asnumpy())
+            images_group.append(img)
+
+        return images_group
+
+    def read_gif(self, video_path, bound=None, fps=25):
+        gif = imageio.get_reader(video_path)
+        max_frame = len(gif) - 1
+
+        images_group = list()
+        frame_indices = self.get_index(bound, fps, max_frame, first_idx=0)
+        for index, frame in enumerate(gif):
+            if index in frame_indices:
+                img = cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB)
+                img = Image.fromarray(img)
+                images_group.append(img)
+
+        return images_group
+
+    def read_frame(self, video_path, bound=None, fps=3):
+        max_frame = len(os.listdir(video_path))
+        images_group = list()
+        frame_indices = self.get_index(bound, fps, max_frame, first_idx=1)  # frame_idx starts from 1
+        for frame_index in frame_indices:
+            img = Image.open(os.path.join(video_path, f'{frame_index:05d}.jpg'))
+            images_group.append(img)
+
+        return images_group
+
+    def qa_template(self, data):
+        question = f"Question: {data['question']}\n"
+        question += 'Options:\n'
+        answer = data['answer']
+        answer_idx = -1
+        for idx, c in enumerate(data['candidates']):
+            question += f"({chr(ord('A') + idx)}) {c}\n"
+            if c == answer:
+                answer_idx = idx
+        question = question.rstrip()
+        answer = f"({chr(ord('A') + answer_idx)}) {answer}"
+        return question, answer
+
+    def __getitem__(self, idx):
+        decord_method = self.decord_method[self.data_list[idx]['data_type']]
+        bound = None
+        if self.data_list[idx]['bound']:
+            bound = (
+                self.data_list[idx]['data']['start'],
+                self.data_list[idx]['data']['end'],
+            )
+        video_path = os.path.join(self.data_list[idx]['prefix'], self.data_list[idx]['data']['video'])
+        image_list = decord_method(video_path, bound)
+        special_tokens = '\n'.join(['Frame{}: <image>'.format(i + 1) for i in range(len(image_list))])
+
+        question, answer = self.qa_template(self.data_list[idx]['data'])
+        question = special_tokens + '\n' + self.prompt + '\n' + question + self.question_prompt
+
+        raw_images = []
+        num_patches_list = []
+        pixel_values = []
+        for image in image_list:
+            raw_images.append(image)
+            if self.dynamic_image_size:
+                patches = dynamic_preprocess(image, image_size=self.input_size,
+                                             use_thumbnail=self.use_thumbnail,
+                                             max_num=self.max_num)
+            else:
+                patches = [image]
+            num_patches_list.append(len(patches))
+            pixel_values.extend([self.transform(patch) for patch in patches])
+
+        pixel_values = torch.stack(pixel_values)
+
+        return {
+            'question': question,
+            'pixel_values': pixel_values,
+            'answer': answer,
+            'num_patches_list': num_patches_list,
+            'task_type': self.data_list[idx]['task_type']
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def check_ans(pred, gt):
+    flag = False
+    pred = pred.replace('Answer: ', '')
+
+    pred_list = pred.lower().split(' ')
+    pred_option, pred_content = pred_list[0], ' '.join(pred_list[1:])
+    gt_list = gt.lower().split(' ')
+    gt_option, gt_content = gt_list[0], ' '.join(gt_list[1:])
+    if gt_content[-1] == '.':
+        gt_content = gt_content[:-1]
+
+    if pred_option.replace('.', '') in gt_option:
+        flag = True
+    elif gt_option in pred_option:
+        flag = True
+
+    return flag
+
+
+def evaluate_chat_model():
+    random.seed(args.seed)
+    prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n'
+    question_prompt = '\nOnly give the best option.'
+
+    vid_dataset = MVBenchDataset(
+        data_dir, data_list,
+        prompt=prompt,
+        question_prompt=question_prompt,
+        num_segments=args.num_segments,
+        input_size=image_size,
+        dynamic_image_size=args.dynamic,
+        use_thumbnail=use_thumbnail,
+        max_num=args.max_num)
+    dataloader = torch.utils.data.DataLoader(
+        dataset=vid_dataset,
+        sampler=InferenceSampler(len(vid_dataset)),
+        batch_size=args.batch_size,
+        num_workers=args.num_workers,
+        pin_memory=True,
+        drop_last=False,
+        collate_fn=partial(collate_fn, tokenizer=tokenizer),
+    )
+
+    outputs = []
+    for _, (pixel_values, questions, answers, num_patches_lists, task_types) in tqdm(enumerate(dataloader)):
+        pixel_values = pixel_values.to(torch.bfloat16).cuda()
+        generation_config = dict(
+            num_beams=args.num_beams,
+            max_new_tokens=1000,
+            min_new_tokens=1,
+            do_sample=True if args.temperature > 0 else False,
+            temperature=args.temperature,
+        )
+        pred = model.chat(
+            tokenizer=tokenizer,
+            pixel_values=pixel_values,
+            num_patches_list=num_patches_lists[0],
+            question=questions[0],
+            generation_config=generation_config,
+            verbose=True
+        )
+        outputs.append({
+            'question': questions[0],
+            'pred': pred,
+            'gt': answers[0],
+            'task_type': task_types[0],
+        })
+    torch.distributed.barrier()
+
+    world_size = torch.distributed.get_world_size()
+    merged_outputs = [None for _ in range(world_size)]
+    torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+    merged_outputs = [json.loads(_) for _ in merged_outputs]
+    merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+    if torch.distributed.get_rank() == 0:
+
+        print(f'Evaluating MVBench ...')
+        correct, total, acc_dict = 0, 0, {}
+        for item in merged_outputs:
+            task_type = item['task_type']
+            pred = item['pred']
+            gt = item['gt']
+            if task_type not in acc_dict:
+                acc_dict[task_type] = [0, 0]  # correct, total
+            acc_dict[task_type][1] += 1
+            total += 1
+
+            if check_ans(pred, gt):
+                acc_dict[task_type][0] += 1
+                correct += 1
+
+        final_res = {}
+        for k, v in acc_dict.items():
+            final_res[k] = v[0] / v[1] * 100
+        final_res['Avg'] = correct / total * 100
+        print(final_res)
+
+        time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+        results_file = f'MVBench_{time_prefix}'
+        output_path = os.path.join(args.out_dir, results_file)
+        with open(f'{output_path}.json', 'w') as f:
+            json.dump(outputs, f)
+        with open(f'{output_path}_result_final.json', 'w') as f:
+            json.dump(final_res, f)
+        print('Results saved to {}'.format(output_path))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='mvbench')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=1)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    parser.add_argument('--num_segments', type=int, default=16)
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()

internvl_chat/eval/pope/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `POPE`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### POPE
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/pope && cd data/pope
+
+# Step 2: Make sure you have downloaded COCO images
+ln -s ../coco/val2014 ./
+wget https://github.com/OpenGVLab/InternVL/releases/download/data/llava_pope_test.jsonl
+
+# Step 3: Download `coco` from POPE
+mkdir -p coco && cd coco
+wget https://github.com/AoiDragon/POPE/raw/e3e39262c85a6a83f26cf5094022a782cb0df58d/output/coco/coco_pope_adversarial.json
+wget https://github.com/AoiDragon/POPE/raw/e3e39262c85a6a83f26cf5094022a782cb0df58d/output/coco/coco_pope_popular.json
+wget https://github.com/AoiDragon/POPE/raw/e3e39262c85a6a83f26cf5094022a782cb0df58d/output/coco/coco_pope_random.json
+cd ../../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/pope
+├── coco
+│   ├── coco_pope_adversarial.json
+│   ├── coco_pope_popular.json
+│   └── coco_pope_random.json
+├── llava_pope_test.jsonl
+└── val2014
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/pope/evaluate_pope.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} pope --dynamic
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default  | Description                                                                                                       |
+| ---------------- | ------ | -------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`     | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'pope'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`  | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`      | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`  | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`  | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/pope/eval_pope.py

+import argparse
+import json
+import os
+
+
+def eval_pope(answers, label_file):
+    label_list = [json.loads(q)['label'] for q in open(label_file, 'r')]
+
+    for answer in answers:
+        text = answer['text']
+
+        # Only keep the first sentence
+        if text.find('.') != -1:
+            text = text.split('.')[0]
+
+        text = text.replace(',', '')
+        words = text.split(' ')
+        if 'No' in words or 'not' in words or 'no' in words:
+            answer['text'] = 'no'
+        else:
+            answer['text'] = 'yes'
+
+    for i in range(len(label_list)):
+        if label_list[i] == 'no':
+            label_list[i] = 0
+        else:
+            label_list[i] = 1
+
+    pred_list = []
+    for answer in answers:
+        if answer['text'] == 'no':
+            pred_list.append(0)
+        else:
+            pred_list.append(1)
+
+    pos = 1
+    neg = 0
+    yes_ratio = pred_list.count(1) / len(pred_list)
+
+    TP, TN, FP, FN = 0, 0, 0, 0
+    for pred, label in zip(pred_list, label_list):
+        if pred == pos and label == pos:
+            TP += 1
+        elif pred == pos and label == neg:
+            FP += 1
+        elif pred == neg and label == neg:
+            TN += 1
+        elif pred == neg and label == pos:
+            FN += 1
+
+    print('TP\tFP\tTN\tFN\t')
+    print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN))
+
+    precision = float(TP) / float(TP + FP)
+    recall = float(TP) / float(TP + FN)
+    f1 = 2 * precision * recall / (precision + recall)
+    acc = (TP + TN) / (TP + TN + FP + FN)
+    print('Accuracy: {}'.format(acc))
+    print('Precision: {}'.format(precision))
+    print('Recall: {}'.format(recall))
+    print('F1 score: {}'.format(f1))
+    print('Yes ratio: {}'.format(yes_ratio))
+    print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio))
+
+    return f1
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--annotation-dir', type=str)
+    parser.add_argument('--question-file', type=str)
+    parser.add_argument('--result-file', type=str)
+    args = parser.parse_args()
+
+    f1_list = []
+    questions = [json.loads(line) for line in open(args.question_file)]
+    questions = {question['question_id']: question for question in questions}
+    answers = json.loads(open(args.result_file).read())
+    for file in os.listdir(args.annotation_dir):
+        assert file.startswith('coco_pope_')
+        assert file.endswith('.json')
+        category = file[10:-5]
+        cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category]
+        print('Category: {}, # samples: {}'.format(category, len(cur_answers)))
+        f1_list.append(eval_pope(cur_answers, os.path.join(args.annotation_dir, file)))
+        print('====================================')
+
+    print(f'Overall F1: {sum(f1_list)/len(f1_list)*100:.2f}')

internvl_chat/eval/pope/evaluate_pope.py

+import argparse
+import itertools
+import json
+import os
+import random
+import re
+import time
+from functools import partial
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'pope': {
+        'root': 'data/pope/val2014',
+        'question': 'data/pope/llava_pope_test.jsonl',
+        'metric': None,
+        'max_new_tokens': 100,
+        'min_new_tokens': 1,
+    }
+}
+
+
+COT_INSTRUCTION = (
+    'Your task is to answer the question below. '
+    "Give step by step reasoning before you answer, and when you're ready to answer, "
+    "please use the format \"Final answer: ..\""
+    '\n\n'
+    'Question:'
+    '\n\n'
+    '{question}'
+)
+
+
+def extract_answer(text):
+    match = re.search(r'(Final answer:|Answer:)\s*(.*)', text, re.IGNORECASE)
+    if match:
+        return match.group(2).strip()
+    return text
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    question_ids = [_['question_id'] for _ in batches]
+    annotations = [_['annotation'] for _ in batches]
+
+    return pixel_values, questions, question_ids, annotations
+
+
+class VQADataset(torch.utils.data.Dataset):
+
+    def __init__(self, root, data, prompt, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6):
+        self.root = root
+        self.data = open(data).readlines()
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        data = json.loads(self.data[idx].strip())
+        image, question, question_id, annotation = data['image'], data[
+            'text'], data['question_id'], data.get('answer', None)
+
+        image = os.path.join(self.root, image)
+        image = Image.open(image).convert('RGB')
+        if self.dynamic_image_size:
+            images = dynamic_preprocess(image, image_size=self.input_size,
+                                        use_thumbnail=self.use_thumbnail,
+                                        max_num=self.max_num)
+        else:
+            images = [image]
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+
+        llava_prompt = 'Answer the question using a single word or phrase.'
+        assert llava_prompt in question
+        question = question.replace(llava_prompt, self.prompt).strip()
+
+        return {
+            'question_id': question_id,
+            'question': question,
+            'pixel_values': pixel_values,
+            'annotation': annotation
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def evaluate_chat_model():
+    prompt = '' if args.cot else 'Answer the question using a single word or phrase.'
+    random.seed(args.seed)
+
+    for ds_name in args.datasets:
+        dataset = VQADataset(
+            root=ds_collections[ds_name]['root'],
+            data=ds_collections[ds_name]['question'],
+            prompt=prompt,
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, question_ids, annotations) in tqdm(enumerate(dataloader)):
+            if args.cot:
+                questions = [COT_INSTRUCTION.format(question=q) for q in questions]
+
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'] if not args.cot else 4096,
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            pred = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=questions[0],
+                generation_config=generation_config,
+                verbose=True
+            )
+            pred_orig = pred
+            if args.cot:
+                pred = extract_answer(pred).strip()
+            answers = [pred]
+
+            for question_id, answer, annotation in zip(question_ids, answers, annotations):
+                outputs.append({
+                    'question_id': question_id,
+                    'text': pred,
+                    'text_orig': pred_orig,
+                    'model_id': args.checkpoint,
+                    'metadata': {},
+                })
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.json'
+            results_file = os.path.join(args.out_dir, results_file)
+            json.dump(merged_outputs, open(results_file, 'w'))
+            print('Results saved to {}'.format(results_file))
+            cmd = 'python eval/pope/eval_pope.py ' \
+                  '--annotation-dir ./data/pope/coco ' \
+                  '--question-file ./data/pope/llava_pope_test.jsonl ' \
+                  '--result-file ' + results_file
+            print(cmd)
+            os.system(cmd)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='pope')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    parser.add_argument('--cot', action='store_true')
+    args = parser.parse_args()
+
+    model_name = '_'.join(args.checkpoint.split('/')[-2:])
+    model_name = f'{model_name}_cot' if args.cot else model_name
+    args.out_dir = os.path.join(args.out_dir, model_name)
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+
+    evaluate_chat_model()

internvl_chat/eval/refcoco/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `RefCOCO`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### RefCOCO/RefCOCO+/RefCOCO-g
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/refcoco && cd data/refcoco
+
+# Step 2: Download converted files
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcoco/refcoco_val.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcoco/refcoco_testA.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcoco/refcoco_testB.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcoco%2B/refcoco%2B_val.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcoco%2B/refcoco%2B_testA.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcoco%2B/refcoco%2B_testB.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcocog/refcocog_val.jsonl
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/refcocog/refcocog_test.jsonl
+
+cd ../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/refcoco
+├── refcocog_test.jsonl
+├── refcocog_val.jsonl
+├── refcoco_testA.jsonl
+├── refcoco+_testA.jsonl
+├── refcoco_testB.jsonl
+├── refcoco+_testB.jsonl
+├── refcoco_val.jsonl
+└── refcoco+_val.jsonl
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/refcoco/evaluate_grounding.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} refcoco --dynamic
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default                                                                                                                               | Description                                                                                                       |
+| ---------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`                                                                                                                                  | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `refcoco_val`, `refcoco_testA`, `refcoco_testB` , `refcoco+_val`, `refcoco+_testA`, `refcoco+_testB`, `refcocog_val`, `refcocog_test` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`                                                                                                                               | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`                                                                                                                                   | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`                                                                                                                               | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`                                                                                                                               | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |