"Initial commit"

cookbooks/utils/agent_function_call.py

+from typing import Union, Tuple, List
+
+from qwen_agent.tools.base import BaseTool, register_tool
+
+
+@register_tool("mobile_use")
+class MobileUse(BaseTool):
+    @property
+    def description(self):
+        return f"""
+Use a touchscreen to interact with a mobile device, and take screenshots.
+* This is an interface to a mobile device with touchscreen. You can perform actions like clicking, typing, swiping, etc.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions.
+* The screen's resolution is {self.display_width_px}x{self.display_height_px}.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
+""".strip()
+
+    parameters = {
+        "properties": {
+            "action": {
+                "description": """
+The action to perform. The available actions are:
+* `key`: Perform a key event on the mobile device.
+    - This supports adb's `keyevent` syntax.
+    - Examples: "volume_up", "volume_down", "power", "camera", "clear".
+* `click`: Click the point on the screen with coordinate (x, y).
+* `long_press`: Press the point on the screen with coordinate (x, y) for specified seconds.
+* `swipe`: Swipe from the starting point with coordinate (x, y) to the end point with coordinates2 (x2, y2).
+* `type`: Input the specified text into the activated input box.
+* `system_button`: Press the system button.
+* `open`: Open an app on the device.
+* `wait`: Wait specified seconds for the change to happen.
+* `terminate`: Terminate the current task and report its completion status.
+""".strip(),
+                "enum": [
+                    "key",
+                    "click",
+                    "long_press",
+                    "swipe",
+                    "type",
+                    "system_button",
+                    "open",
+                    "wait",
+                    "terminate",
+                ],
+                "type": "string",
+            },
+            "coordinate": {
+                "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=click`, `action=long_press`, and `action=swipe`.",
+                "type": "array",
+            },
+            "coordinate2": {
+                "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=swipe`.",
+                "type": "array",
+            },
+            "text": {
+                "description": "Required only by `action=key`, `action=type`, and `action=open`.",
+                "type": "string",
+            },
+            "time": {
+                "description": "The seconds to wait. Required only by `action=long_press` and `action=wait`.",
+                "type": "number",
+            },
+            "button": {
+                "description": "Back means returning to the previous interface, Home means returning to the desktop, Menu means opening the application background menu, and Enter means pressing the enter. Required only by `action=system_button`",
+                "enum": [
+                    "Back",
+                    "Home",
+                    "Menu",
+                    "Enter",
+                ],
+                "type": "string",
+            },
+            "status": {
+                "description": "The status of the task. Required only by `action=terminate`.",
+                "type": "string",
+                "enum": ["success", "failure"],
+            },
+        },
+        "required": ["action"],
+        "type": "object",
+    }
+
+    def __init__(self, cfg=None):
+        self.display_width_px = cfg["display_width_px"]
+        self.display_height_px = cfg["display_height_px"]
+        super().__init__(cfg)
+
+    def call(self, params: Union[str, dict], **kwargs):
+        params = self._verify_json_format_args(params)
+        action = params["action"]
+        if action == "key":
+            return self._key(params["text"])
+        elif action == "click":
+            return self._click(
+                coordinate=params["coordinate"]
+            )
+        elif action == "long_press":
+            return self._long_press(
+                coordinate=params["coordinate"], time=params["time"]
+            )
+        elif action == "swipe":
+            return self._swipe(
+                coordinate=params["coordinate"], coordinate2=params["coordinate2"]
+            )
+        elif action == "type":
+            return self._type(params["text"])
+        elif action == "system_button":
+            return self._system_button(params["button"])
+        elif action == "open":
+            return self._open(params["text"])
+        elif action == "wait":
+            return self._wait(params["time"])
+        elif action == "terminate":
+            return self._terminate(params["status"])
+        else:
+            raise ValueError(f"Unknown action: {action}")
+
+    def _key(self, text: str):
+        raise NotImplementedError()
+        
+    def _click(self, coordinate: Tuple[int, int]):
+        raise NotImplementedError()
+
+    def _long_press(self, coordinate: Tuple[int, int], time: int):
+        raise NotImplementedError()
+
+    def _swipe(self, coordinate: Tuple[int, int], coordinate2: Tuple[int, int]):
+        raise NotImplementedError()
+
+    def _type(self, text: str):
+        raise NotImplementedError()
+
+    def _system_button(self, button: str):
+        raise NotImplementedError()
+
+    def _open(self, text: str):
+        raise NotImplementedError()
+
+    def _wait(self, time: int):
+        raise NotImplementedError()
+
+    def _terminate(self, status: str):
+        raise NotImplementedError()
+    
+@register_tool("computer_use")
+class ComputerUse(BaseTool):
+    @property
+    def description(self):
+        return f"""
+Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.
+* The screen's resolution is {self.display_width_px}x{self.display_height_px}.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges.
+""".strip()
+
+    parameters = {
+        "properties": {
+            "action": {
+                "description": """
+The action to perform. The available actions are:
+* `key`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order.
+* `type`: Type a string of text on the keyboard.
+* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
+* `left_click`: Click the left mouse button at a specified (x, y) pixel coordinate on the screen.
+* `left_click_drag`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
+* `right_click`: Click the right mouse button at a specified (x, y) pixel coordinate on the screen.
+* `middle_click`: Click the middle mouse button at a specified (x, y) pixel coordinate on the screen.
+* `double_click`: Double-click the left mouse button at a specified (x, y) pixel coordinate on the screen.
+* `triple_click`: Triple-click the left mouse button at a specified (x, y) pixel coordinate on the screen (simulated as double-click since it's the closest action).
+* `scroll`: Performs a scroll of the mouse scroll wheel.
+* `hscroll`: Performs a horizontal scroll (mapped to regular scroll).
+* `wait`: Wait specified seconds for the change to happen.
+* `terminate`: Terminate the current task and report its completion status.
+* `answer`: Answer a question.
+""".strip(),
+                "enum": [
+                    "key",
+                    "type",
+                    "mouse_move",
+                    "left_click",
+                    "left_click_drag",
+                    "right_click",
+                    "middle_click",
+                    "double_click",
+                    "triple_click",
+                    "scroll",
+                    "hscroll",
+                    "wait",
+                    "terminate",
+                    "answer",
+                ],
+                "type": "string",
+            },
+            "keys": {
+                "description": "Required only by `action=key`.",
+                "type": "array",
+            },
+            "text": {
+                "description": "Required only by `action=type` and `action=answer`.",
+                "type": "string",
+            },
+            "coordinate": {
+                "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to.",
+                "type": "array",
+            },
+            "pixels": {
+                "description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll` and `action=hscroll`.",
+                "type": "number",
+            },
+            "time": {
+                "description": "The seconds to wait. Required only by `action=wait`.",
+                "type": "number",
+            },
+            "status": {
+                "description": "The status of the task. Required only by `action=terminate`.",
+                "type": "string",
+                "enum": ["success", "failure"],
+            },
+        },
+        "required": ["action"],
+        "type": "object",
+    }
+
+    def __init__(self, cfg=None):
+        self.display_width_px = cfg["display_width_px"]
+        self.display_height_px = cfg["display_height_px"]
+        super().__init__(cfg)
+
+    def call(self, params: Union[str, dict], **kwargs):
+        params = self._verify_json_format_args(params)
+        action = params["action"]
+        if action in ["left_click", "right_click", "middle_click", "double_click","triple_click"]:
+            return self._mouse_click(action)
+        elif action == "key":
+            return self._key(params["keys"])
+        elif action == "type":
+            return self._type(params["text"])
+        elif action == "mouse_move":
+            return self._mouse_move(params["coordinate"])
+        elif action == "left_click_drag":
+            return self._left_click_drag(params["coordinate"])
+        elif action == "scroll":
+            return self._scroll(params["pixels"])
+        elif action == "hscroll":
+            return self._hscroll(params["pixels"])
+        elif action == "answer":
+            return self._answer(params["text"])
+        elif action == "wait":
+            return self._wait(params["time"])
+        elif action == "terminate":
+            return self._terminate(params["status"])
+        else:
+            raise ValueError(f"Invalid action: {action}")
+
+    def _mouse_click(self, button: str):
+        raise NotImplementedError()
+
+    def _key(self, keys: List[str]):
+        raise NotImplementedError()
+
+    def _type(self, text: str):
+        raise NotImplementedError()
+
+    def _mouse_move(self, coordinate: Tuple[int, int]):
+        raise NotImplementedError()
+
+    def _left_click_drag(self, coordinate: Tuple[int, int]):
+        raise NotImplementedError()
+
+    def _scroll(self, pixels: int):
+        raise NotImplementedError()
+
+    def _hscroll(self, pixels: int):
+        raise NotImplementedError()
+
+    def _answer(self, text: str):
+        raise NotImplementedError()
+
+    def _wait(self, time: int):
+        raise NotImplementedError()
+
+    def _terminate(self, status: str):
+        raise NotImplementedError()
\ No newline at end of file

cookbooks/utils/multimodal_coding/take_screenshot.py

+import os
+from playwright.sync_api import sync_playwright
+import argparse
+from PIL import Image
+import time
+
+
+def take_screenshot(url, output_file="screenshot.png"):
+    # Convert local path to file:// URL if it's a file
+    if os.path.exists(url):
+        url = "file://" + os.path.abspath(url)
+
+    try:
+        with sync_playwright() as p:
+            # Choose a browser, e.g., Chromium, Firefox, or WebKit
+            browser = p.chromium.launch(headless=True, args=["--disable-web-security"])
+            page = browser.new_page()
+
+            # Navigate to the URL
+            page.goto(url, timeout=60000)
+            # page.wait_for_timeout(1000)  # give it 1 second to paint
+
+            # Take the screenshot
+            page.screenshot(path=output_file, full_page=True, animations="disabled", timeout=60000)
+
+            browser.close()
+    except Exception as e:
+        print(f"Failed to take screenshot due to: {e}. Generating a blank image.")
+        # Generate a blank image
+        img = Image.new("RGB", (1280, 960), color="white")
+        img.save(output_file)
+
+
+if __name__ == "__main__":
+
+    # Initialize the parser
+    parser = argparse.ArgumentParser(description="Process two path strings.")
+
+    # Define the arguments
+    parser.add_argument("--html", type=str)
+    parser.add_argument("--png", type=str)
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    take_screenshot(args.html, args.png)

cookbooks/utils/multimodal_coding/test_mmcode.py

+# Standard library imports
+import base64
+import gzip
+import importlib
+import json
+import multiprocessing
+import os
+import platform
+import re
+import signal
+import sys
+import typing
+import faulthandler
+from datetime import datetime
+from io import BytesIO, StringIO
+from unittest.mock import mock_open, patch
+
+# Third-party imports
+import numpy as np
+from PIL import Image
+
+
+# from pyext import RuntimeModule
+# Fix for higher versions of python
+def create_module_from_string(name, code):
+    spec = importlib.util.spec_from_loader(name, loader=None)
+    module = importlib.util.module_from_spec(spec)
+    exec(code, module.__dict__)
+    return module
+
+
+from enum import Enum
+
+
+class CODE_TYPE(Enum):
+    call_based = 0
+    standard_input = 1
+
+
+# stuff for setting up signal timer
+class TimeoutException(Exception):
+    pass
+
+
+def timeout_handler(signum, frame):
+    print("alarm went off")
+    # return
+    raise TimeoutException
+
+
+signal.signal(signal.SIGALRM, timeout_handler)
+timeout = 20  # seconds
+
+
+# used to capture stdout as a list
+# from https://stackoverflow.com/a/16571630/6416660
+# alternative use redirect_stdout() from contextlib
+class Capturing(list):
+    def __enter__(self):
+        self._stdout = sys.stdout
+        sys.stdout = self._stringio = StringIO()
+        # Make closing the StringIO a no-op
+        self._stringio.close = lambda x: 1
+        return self
+
+    def __exit__(self, *args):
+        self.extend(self._stringio.getvalue().splitlines())
+        del self._stringio  # free up some memory
+        sys.stdout = self._stdout
+
+
+def run_test(sample, test=None, debug=True):
+    """
+    if test(generated_code) is not None it'll try to run the code.
+    otherwise it'll just return an input and output pair.
+    """
+    # Disable functionalities that can make destructive changes to the test.
+    reliability_guard()
+
+    if debug:
+        print(f"start = {datetime.now().time()}")
+
+    try:
+        in_outs = json.loads(sample["input_output"])
+    except ValueError:
+        in_outs = None
+    if in_outs:
+        if in_outs.get("fn_name") is None:
+            which_type = CODE_TYPE.standard_input  # Standard input
+            method_name = None
+        else:
+            which_type = CODE_TYPE.call_based  # Call-based
+            method_name = in_outs["fn_name"]
+
+    if debug:
+        print(f"loaded input_output = {datetime.now().time()}")
+
+    if test is None:
+        return in_outs
+    elif test is not None:
+        results = []
+        sol = "import sys\nimport time\nimport itertools\nfrom itertools import accumulate, product, permutations, combinations\nimport collections\nfrom collections import Counter, OrderedDict, deque, defaultdict, ChainMap\nfrom functools import lru_cache\nimport math\nfrom math import sqrt, sin, cos, tan, ceil, fabs, floor, gcd, exp, log, log2\nimport fractions\nfrom typing import List, Tuple\nimport numpy as np\nimport random\nimport heapq\nfrom heapq import *\n"
+        if debug:
+            print(f"loading test code = {datetime.now().time()}")
+
+        if which_type == CODE_TYPE.call_based:
+            sol += test
+            if debug:
+                print(f"sol = {sol}")
+            signal.alarm(timeout)
+            try:
+                # tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol)
+                # An ugly fix for Gemini: it keeps generating code that uses the `__name__`
+                sol = sol.replace("__main__", "tmp_sol")
+                tmp_sol = create_module_from_string("tmp_sol", sol)
+                if "class Solution" not in test:
+                    tmp = tmp_sol
+                else:
+                    tmp = tmp_sol.Solution()
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f"type 0 compilation error = {e}")
+                results.append(-2)
+                return results
+            signal.alarm(0)
+
+        elif which_type == CODE_TYPE.standard_input:
+            # sol
+            tmp_test = test.split("\n")
+
+            new_test = []
+            for x in tmp_test:
+                if (not x.startswith("from ")) and (not x.startswith("import ")):
+                    new_test.append("\t" + x + "\n")
+                else:
+                    new_test.append(x + "\n")
+            tmp_test = new_test
+
+            new_test = ""
+            started = False
+            for i in tmp_test:
+                if i.startswith("\t") and not started:
+                    new_test += "stdin = sys.stdin\nstdout = sys.stdout\n"
+                    new_test += "def code():\n"
+                    new_test += i
+                    started = True
+                elif started and ((i.startswith("from ")) or (i.startswith("import "))):
+                    new_test += "\t" + i
+                else:
+                    new_test += i
+            tmp_test = new_test
+
+            sol += tmp_test
+            if debug:
+                print(f"sol = {sol}")
+            method_name = "code"
+            signal.alarm(timeout)
+            try:
+                # tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol)
+                # An ugly fix for Gemini: it keeps generating code that uses the `__name__`
+                sol = sol.replace("__main__", "tmp_sol")
+                tmp_sol = create_module_from_string("tmp_sol", sol)
+                tmp = tmp_sol
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f"type 1 compilation error = {e}")
+                results.append(-2)
+                return results
+            signal.alarm(0)
+        if debug:
+            print(f"get method = {datetime.now().time()}")
+
+        try:
+            method = getattr(tmp, method_name)  # get_attr second arg must be str
+        except:
+            signal.alarm(0)
+            e = sys.exc_info()
+            print(f"unable to get function error = {e}")
+            results.append(-2)
+            return results
+
+        for index, inputs in enumerate(in_outs["inputs"]):
+            # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list)
+            try:
+                if isinstance(inputs[0], dict):
+                    inputs = [{int(k): v for k, v in inputs[0].items()}]
+            except:
+                True
+            try:
+                if isinstance(in_outs["outputs"][index], dict):
+                    in_outs["outputs"][index] = [{int(k): v for k, v in in_outs["outputs"][index].items()}]
+            except:
+                True
+            try:
+                if isinstance(in_outs["outputs"][index][0], dict):
+                    in_outs["outputs"][index] = [{int(k): v for k, v in in_outs["outputs"][index][0].items()}]
+            except:
+                True
+
+            if debug:
+                print(f"time: {datetime.now().time()} testing index = {index}  inputs = {inputs}, {type(inputs)}. type = {which_type}")
+            if which_type == CODE_TYPE.call_based:  # Call-based
+                signal.alarm(timeout)
+                faulthandler.enable()
+                try:
+                    output = method(*inputs)
+
+                    # ground truth sequences are not tuples
+                    if isinstance(output, tuple):
+                        output = list(output)
+
+                    tmp_result = output == in_outs["outputs"][index]
+                    if isinstance(in_outs["outputs"][index], list) and in_outs["outputs"][index]:
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index][0])
+
+                    # ground truth sequences are not tuples
+                    try:
+                        if isinstance(output[0], tuple):
+                            tmp_result = tmp_result or ([list(x) for x in output] == in_outs["outputs"][index][0])
+                    except:
+                        True
+                    results.append(tmp_result)
+
+                    # reset the alarm
+                    signal.alarm(0)
+                except Exception as e:
+                    signal.alarm(0)
+                    faulthandler.disable()
+                    print(f"Standard input runtime error or time limit exceeded error = {e}")
+                    results.append(-1)
+                    continue
+                faulthandler.disable()
+                signal.alarm(0)
+                if debug:
+                    print(f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+            elif which_type == CODE_TYPE.standard_input:  # Standard input
+                faulthandler.enable()
+                signal.alarm(timeout)
+                passed = False
+
+                if isinstance(inputs, list):
+                    inputs = "\n".join(inputs)
+                if isinstance(in_outs["outputs"][index], list):
+                    in_outs["outputs"][index] = "\n".join(in_outs["outputs"][index])
+
+                with Capturing() as output:
+                    try:
+                        call_method(method, inputs)
+                        # reset the alarm
+                        signal.alarm(0)
+                        passed = True
+                    except Exception as e:
+                        # runtime error or took too long
+                        signal.alarm(0)
+                        print(f"Call-based runtime error or time limit exceeded error = {repr(e)}{e}")
+                        results.append(-1)
+                    signal.alarm(0)
+
+                if not passed:
+                    if debug:
+                        nl = "\n"
+                        if not isinstance(inputs, list):
+                            print(
+                                f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"
+                            )
+                        else:
+                            print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+                    continue
+
+                if passed and debug:
+                    print(f"==> output = {output}, test outputs = {in_outs['outputs'][index]}")
+
+                if custom_compare_(output, in_outs["outputs"][index]):
+                    tmp_result = True
+                    results.append(tmp_result)
+                    continue
+
+                # ground truth sequences are expressed as lists not tuples
+                if isinstance(output, tuple):
+                    output = list(output)
+
+                tmp_result = False
+                try:
+                    tmp_result = output == [in_outs["outputs"][index]]
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                        if isinstance(output[0], str):
+                            tmp_result = tmp_result or ([e.strip() for e in output] == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check1 exception = {e}")
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try one more time without \n
+                if isinstance(in_outs["outputs"][index], list):
+                    for tmp_index, i in enumerate(in_outs["outputs"][index]):
+                        in_outs["outputs"][index][tmp_index] = i.split("\n")
+                        in_outs["outputs"][index][tmp_index] = [x.strip() for x in in_outs["outputs"][index][tmp_index] if x]
+                else:
+                    in_outs["outputs"][index] = in_outs["outputs"][index].split("\n")
+                    in_outs["outputs"][index] = list(filter(len, in_outs["outputs"][index]))
+                    in_outs["outputs"][index] = list(map(lambda x: x.strip(), in_outs["outputs"][index]))
+
+                try:
+                    tmp_result = output == [in_outs["outputs"][index]]
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check2 exception = {e}")
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    output = list(filter(len, output))
+
+                if debug:
+                    nl = "\n"
+                    if not isinstance(inputs, list):
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+                    else:
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                try:
+                    tmp_result = output == [in_outs["outputs"][index]]
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check3 exception = {e}")
+                    pass
+
+                try:
+                    output_float = [float(e) for e in output]
+                    gt_float = [float(e) for e in in_outs["outputs"][index]]
+                    tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float))
+                except Exception as e:
+                    pass
+                try:
+                    if isinstance(output[0], list):
+                        output_float = [float(e) for e in output[0]]
+                        gt_float = [float(e) for e in in_outs["outputs"][index][0]]
+                        tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float))
+                except Exception as e:
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the stuff into split up list
+                if isinstance(in_outs["outputs"][index], list):
+                    for tmp_index, i in enumerate(in_outs["outputs"][index]):
+                        in_outs["outputs"][index][tmp_index] = set(i.split())
+                else:
+                    in_outs["outputs"][index] = set(in_outs["outputs"][index].split())
+
+                try:
+                    tmp_result = output == in_outs["outputs"][index]
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check4 exception = {e}")
+                    continue
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = i.split()
+                    output = list(filter(len, output))
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = set(i)
+                else:
+                    output = output.split()
+                    output = list(filter(len, output))
+                    output = set(output)
+
+                try:
+                    tmp_result = set(frozenset(s) for s in output) == set(frozenset(s) for s in in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check5 exception = {e}")
+
+                # if they are all numbers, round so that similar numbers are treated as identical
+                try:
+                    tmp_result = tmp_result or (set(frozenset(round(float(t), 3) for t in s) for s in output) == set(frozenset(round(float(t), 3) for t in s) for s in in_outs["outputs"][index]))
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check6 exception = {e}")
+
+                if tmp_result == True and debug:
+                    print("PASSED")
+
+                results.append(tmp_result)
+
+                if debug:
+                    nl = "\n"
+                    if not isinstance(inputs, list):
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+                    else:
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+
+    sys.stdout.flush()
+    return results
+
+
+def custom_compare_(output, ground_truth):
+
+    if isinstance(output, list):
+        output_1 = "\n".join(output)
+        if stripped_string_compare(output_1, ground_truth):
+            return True
+
+    if isinstance(output, list):
+        output_2 = [o.lstrip().rstrip() for o in output]
+        output_2 = "\n".join(output_2)
+        if stripped_string_compare(output_2, ground_truth):
+            return True
+
+    return False
+
+
+def stripped_string_compare(s1, s2):
+    s1 = s1.lstrip().rstrip()
+    s2 = s2.lstrip().rstrip()
+    return s1 == s2
+
+
+def call_method(method, inputs):
+
+    if isinstance(inputs, list):
+        inputs = "\n".join(inputs)
+
+    inputs_line_iterator = iter(inputs.split("\n"))
+
+    # sys.setrecursionlimit(10000)
+
+    # @patch('builtins.input', side_effect=inputs.split("\n"))
+    @patch("builtins.open", mock_open(read_data=inputs))
+    @patch("sys.stdin", StringIO(inputs))
+    @patch("sys.stdin.readline", lambda *args: next(inputs_line_iterator))
+    @patch("sys.stdin.readlines", lambda *args: inputs.split("\n"))
+    @patch("sys.stdin.read", lambda *args: inputs)
+    # @patch('sys.stdout.write', print)
+    def _inner_call_method(_method):
+        try:
+            return _method()
+        except SystemExit as e:
+            pass
+        finally:
+            pass
+
+    return _inner_call_method(method)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    # __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None
+
+
+def extract_last_code_block(text):
+    """Extract the last Python code block from the text"""
+    code_blocks = re.findall(r"```python(.*?)```", text, re.DOTALL)
+    if code_blocks:
+        return code_blocks[-1].strip()
+    return None
+
+
+def check_correctness(sample, generation, timeout=20, debug=True):
+    """Check correctness of code generation with a global timeout (from MMCode eval.py)"""
+
+    def _temp_run(sample, generation, debug, result):
+        result.append(run_test(sample, test=generation, debug=debug))
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+    p = multiprocessing.Process(target=_temp_run, args=(sample, generation, debug, result))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+    if not result:
+        in_outs = json.loads(sample["input_output"])
+        # consider that all tests failed
+        result = [[-1 for i in range(len(in_outs["inputs"]))]]
+        print(f"global timeout")
+        if debug:
+            print(f"global timeout")
+    return result[0]
+
+
+def evaluate_mmcode_solution(problem, solution_code, timeout=20, debug=True):
+    """
+    Evaluate a solution using the exact MMCode testing framework
+    """
+    if not problem or not solution_code:
+        print("❌ Missing problem or solution")
+        return None
+
+    print(f"\n🧪 TESTING SOLUTION with MMCode framework:")
+    print("-" * 50)
+
+    # Extract the Python code from the solution (handling ```python blocks)
+    extracted_code = extract_last_code_block(solution_code)
+    if extracted_code:
+        clean_solution = extracted_code
+    else:
+        # If no code block found, use the entire solution
+        clean_solution = solution_code.strip()
+
+    if not clean_solution:
+        print("❌ No valid Python code found in solution")
+        return None
+
+    print(f"📝 Testing code...")
+
+    try:
+        # Run the test using MMCode's exact testing framework
+        curr_res = check_correctness(problem, clean_solution, timeout=timeout, debug=debug)
+
+        if curr_res is None:
+            print("❌ Test execution failed")
+            return None
+
+        # Fix numpy arrays and booleans (from eval.py)
+        fixed = []
+        for e in curr_res:
+            if isinstance(e, np.ndarray):
+                e = e.item(0)
+            if isinstance(e, np.bool_):
+                e = bool(e)
+            fixed.append(e)
+        curr_res = fixed
+
+        # Analyze results
+        total_tests = len(curr_res)
+        passed_tests = sum(1 for result in curr_res if result == 1)
+        failed_tests = sum(1 for result in curr_res if result == 0)
+        error_tests = sum(1 for result in curr_res if result == -1)
+        timeout_tests = sum(1 for result in curr_res if result == -2)
+
+        print(f"\n📊 DETAILED RESULTS:")
+        for i, result in enumerate(curr_res):
+            if result == 1:
+                print(f"✅ Test {i+1}: PASSED")
+            elif result == 0:
+                print(f"❌ Test {i+1}: FAILED")
+            elif result == -1:
+                print(f"💥 Test {i+1}: ERROR")
+            elif result == -2:
+                print(f"⏱️  Test {i+1}: TIMEOUT")
+
+        print(f"\n📈 SUMMARY:")
+        print(f"✅ Passed: {passed_tests}")
+        print(f"❌ Failed: {failed_tests}")
+        print(f"💥 Errors: {error_tests}")
+        print(f"⏱️  Timeouts: {timeout_tests}")
+        print(f"📋 Total: {total_tests}")
+
+        accuracy = passed_tests / total_tests if total_tests > 0 else 0
+
+        evaluation = {
+            "total_tests": total_tests,
+            "passed_tests": passed_tests,
+            "failed_tests": failed_tests,
+            "error_tests": error_tests,
+            "timeout_tests": timeout_tests,
+            "accuracy": accuracy,
+            "test_results": curr_res,
+            "all_passed": np.all([r == 1 for r in curr_res]),
+        }
+
+        print(f"\n🎯 FINAL SCORE: {passed_tests}/{total_tests} ({accuracy:.1%})")
+
+        if evaluation["all_passed"]:
+            print(f"🏆 PERFECT: All tests passed!")
+        else:
+            print(f"⚠️  NEEDS IMPROVEMENT: success rate = {accuracy:.1%}")
+
+        return evaluation
+
+    except Exception as e:
+        print(f"❌ Evaluation error: {str(e)}")
+        import traceback
+
+        traceback.print_exc()
+        return None
+
+
+def convert_base64_to_pil_image(base64_str: str) -> Image:
+    """Convert base64 string to PIL Image"""
+    image_data = base64.b64decode(base64_str)
+    image = Image.open(BytesIO(image_data))
+    return image
+
+
+def load_mmcode_problems(file_path):
+    """Load problems from the downloaded JSONL file and convert images to PIL"""
+    problems = []
+    try:
+        with gzip.open(file_path, "rt") as f:
+            for line in f:
+                problem = json.loads(line)
+
+                # Convert base64 images to PIL Images (images are guaranteed to exist)
+                pil_images = []
+                for img_base64 in problem["images"]:
+                    try:
+                        pil_img = convert_base64_to_pil_image(img_base64)
+                        pil_images.append(pil_img)
+                    except Exception as e:
+                        print(f"Warning: Failed to decode image: {e}")
+                        continue
+                problem["images"] = pil_images
+
+                problems.append(problem)
+
+        print(f"✅ Loaded {len(problems)} problems")
+        return problems
+    except Exception as e:
+        print(f"❌ Failed to load problems: {e}")
+        return []
+
+
+def parse_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="MMCode Evaluation Script")
+    parser.add_argument("--problem_file", type=str, default="mmcode_test.jsonl.gz", required=False, help="Path to the JSON file containing problems")
+    parser.add_argument("--problem_index", type=int, default=0, help="Index of the problem to evaluate (default: 0)")
+    parser.add_argument("--solution_file", type=str, required=True, help="Path to the text file containing the solution code")
+    parser.add_argument("--timeout", type=int, default=20, help="Timeout in seconds for each test case (default: 20)")
+    parser.add_argument("--debug", action="store_true", help="Enable debug mode with detailed output")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    problems = load_mmcode_problems(args.problem_file)
+
+    evaluate_mmcode_solution(
+        problem=problems[args.problem_index],
+        solution_code=open(args.solution_file).read(),
+    )

evaluation/MathVision/README.md

+# MathVision Benchmark Evaluation
+
+This directory contains the implementation for evaluating vision-language models on the MathVision benchmark using vLLM for high-speed inference.
+
+## Overview
+
+MathVision is a mathematical visual reasoning benchmark that evaluates models' ability to solve mathematical problems based on visual information. This implementation provides:
+
+- **High-speed inference** using vLLM with automatic batch optimization
+- **Two-stage evaluation** using rule-based and GPT-4o-based answer extraction
+- **Support for thinking models** with extended reasoning capabilities
+- **Modular code structure** for easy maintenance and extension
+
+## Project Structure
+
+```
+MathVision/
+├── run_mathv.py          # Main script for inference and evaluation
+├── dataset_utils.py      # Dataset loading and preprocessing utilities
+├── eval_utils.py         # Evaluation logic and answer extraction
+├── common_utils.py       # Common utilities for image processing, file I/O
+├── infer_instruct.sh     # Inference script for instruct models
+├── infer_think.sh        # Inference script for thinking models
+├── eval_instruct.sh      # Evaluation script for instruct model results
+├── eval_think.sh         # Evaluation script for thinking model results
+├── requirements.txt      # Python dependencies
+└── README.md            # This file
+```
+
+## Requirements
+
+### Python Dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+Key dependencies:
+- `vllm` - High-speed LLM inference engine
+- `transformers` - HuggingFace transformers
+- `qwen_vl_utils` - Qwen VL utilities for vision processing
+- `pandas`, `numpy` - Data processing
+- `Pillow` - Image processing
+- `latex2sympy2` - LaTeX to symbolic math conversion (optional)
+- `openpyxl` - Excel file handling
+- `requests` - API calls for evaluation
+
+### Environment Variables
+
+For evaluation, you need to set up API credentials for the judge model:
+
+**Option 1: DashScope API (Recommended)**
+```bash
+export CHATGPT_DASHSCOPE_API_KEY="your-api-key"
+export DASHSCOPE_API_BASE="https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions"
+```
+
+**Option 2: Custom OpenAI-compatible API**
+```bash
+export MIT_SPIDER_TOKEN="your-api-key"
+export MIT_SPIDER_URL="your-api-endpoint"
+```
+
+## Quick Start
+
+### 1. Inference
+
+Run inference on MathVision dataset using an instruct model:
+
+```bash
+bash infer_instruct.sh
+```
+
+Or customize the inference:
+
+```bash
+python run_mathv.py infer \
+    --model-path /path/to/Qwen3-VL-Instruct \
+    --data-dir /path/to/data \
+    --dataset MathVision \
+    --output-file results/predictions.jsonl \
+    --max-new-tokens 32768 \
+    --temperature 0.7 \
+    --top-p 0.8 \
+    --top-k 20 \
+    --repetition-penalty 1.0 \
+    --presence-penalty 1.5
+```
+
+For thinking models with extended reasoning:
+
+```bash
+bash infer_think.sh
+```
+
+### 2. Evaluation
+
+Evaluate the inference results using GPT-4o as a judge:
+
+```bash
+bash eval_instruct.sh
+```
+
+Or customize the evaluation:
+
+```bash
+python run_mathv.py eval \
+    --data-dir /path/to/data \
+    --input-file results/predictions.jsonl \
+    --output-file results/evaluation.csv \
+    --dataset MathVision \
+    --eval-model gpt-4o-2024-05-13 \
+    --api-type dash \
+    --nproc 16
+```
+
+## Detailed Usage
+
+### Inference Mode
+
+**Basic Arguments:**
+- `--model-path`: Path to the Qwen3-VL model directory (required)
+- `--data-dir`: Directory to store/load MathVision dataset (required)
+- `--dataset`: Dataset name (default: `MathVision`)
+  - `MathVision`: Full dataset with ~3,000 samples
+  - `MathVision_MINI`: Mini version for quick testing
+- `--output-file`: Path to save inference results in JSONL format (required)
+
+**vLLM Arguments:**
+- `--tensor-parallel-size`: Number of GPUs for tensor parallelism (default: auto-detect)
+- `--gpu-memory-utilization`: GPU memory utilization ratio, 0.0-1.0 (default: 0.9)
+- `--max-model-len`: Maximum model context length (default: 128000)
+- `--max-images-per-prompt`: Maximum images per prompt (default: 10)
+
+**Generation Parameters:**
+- `--max-new-tokens`: Maximum tokens to generate (default: 32768)
+- `--temperature`: Sampling temperature (default: 0.7)
+- `--top-p`: Top-p sampling (default: 0.8)
+- `--top-k`: Top-k sampling (default: 20)
+- `--repetition-penalty`: Repetition penalty (default: 1.0)
+- `--presence-penalty`: Presence penalty to reduce repetition (default: 1.5)
+
+**Advanced Options:**
+- `--use-cot`: Enable Chain-of-Thought prompting for better reasoning
+- `--cot-prompt`: Custom CoT prompt (default: " Let's think step by step.")
+- `--num-samples`: Number of samples to process (optional, for testing)
+
+### Evaluation Mode
+
+**Basic Arguments:**
+- `--data-dir`: Directory containing MathVision dataset (required)
+- `--input-file`: Inference results file in JSONL format (required)
+- `--output-file`: Path to save evaluation results in CSV format (required)
+- `--dataset`: Dataset name, must match inference (default: `MathVision`)
+
+**Judge Model Arguments:**
+- `--eval-model`: Judge model name (default: `gpt-4o`)
+  - Options: `gpt-4o`, `gpt-4o-2024-05-13`, `gpt-3.5-turbo-0125`, etc.
+- `--api-type`: API service type (default: `dash`)
+  - `dash`: DashScope API (Alibaba Cloud)
+  - `mit`: Custom OpenAI-compatible API
+- `--nproc`: Number of parallel workers for evaluation (default: 4)
+
+## Output Files
+
+### Inference Output
+
+The inference script generates a JSONL file where each line contains:
+
+```json
+{
+  "question_id": 123,
+  "annotation": {
+    "index": 123,
+    "question": "What is the area of the triangle?",
+    "answer": "12",
+    "category": "Geometry",
+    "choices": "[]",
+    ...
+  },
+  "task": "MathVision",
+  "result": {
+    "gen": "The final answer",
+    "gen_raw": "Raw model output including thinking process"
+  },
+  "messages": [...]
+}
+```
+
+### Evaluation Output
+
+The evaluation script generates multiple files:
+
+1. **Intermediate results** (`*_eval_results.xlsx`): Raw predictions with metadata
+2. **Detailed evaluation** (`*_eval_results_eval.xlsx`): Results with extracted answers
+   - Columns: `index`, `question`, `prediction`, `answer`, `res` (extracted), `log`, `extract_model`, `extract_flag`, `category`
+3. **Score summary** (`*_eval_results_eval_score.csv`): Accuracy by category
+
+Example score summary:
+```
+Subject         | tot | prefetch | hit | prefetch_rate | acc
+----------------|-----|----------|-----|---------------|------
+Overall         | 3000| 2400     | 2100| 80.0          | 70.0
+Algebra         | 800 | 640      | 560 | 80.0          | 70.0
+Geometry        | 750 | 600      | 525 | 80.0          | 70.0
+```
+
+## Model-Specific Configurations
+
+### Instruct Models (e.g., Qwen3-VL-2B-Instruct, Qwen3-VL-30B-Instruct)
+
+Use standard parameters for balanced performance:
+
+```bash
+--max-new-tokens 32768
+--temperature 0.7
+--top-p 0.8
+--top-k 20
+--repetition-penalty 1.0
+--presence-penalty 1.5
+```
+
+### Thinking Models (e.g., Qwen3-VL-4B-Thinking, Qwen3-VL-30B-Thinking)
+
+Use extended parameters for deeper reasoning:
+
+```bash
+--max-new-tokens 40960
+--temperature 1.0
+--top-p 0.95
+--top-k 20
+--repetition-penalty 1.0
+--presence-penalty 0.0
+```
+
+**Note:** Thinking models output reasoning steps wrapped in `<think>...</think>` tags. The evaluation automatically extracts the final answer after `</think>`.
+
+## Performance Tips
+
+1. **GPU Memory**: Adjust `--gpu-memory-utilization` based on your GPU:
+   - 0.9: Recommended for most cases
+   - 0.95: For maximum throughput (may cause OOM)
+   - 0.7-0.8: If experiencing OOM errors
+
+2. **Batch Size**: vLLM automatically optimizes batch size based on available memory
+
+3. **Tensor Parallelism**: Use `--tensor-parallel-size` for large models:
+   - 2B/4B models: 1-2 GPUs
+   - 7B/14B models: 2-4 GPUs
+   - 30B+ models: 4-8 GPUs
+
+4. **Context Length**: Reduce `--max-model-len` if memory is limited:
+   - 128000: Default, works well for most cases
+   - 64000: Reduces memory usage by ~40%
+
+5. **Image Resolution**: MathVision uses optimized resolution (768×28×28 to 5120×28×28)
+   - Lower min_pixels for faster processing
+   - Higher max_pixels for better accuracy on complex diagrams
+
+## Troubleshooting
+
+### Common Issues
+
+**1. CUDA Out of Memory**
+```bash
+# Reduce GPU memory utilization
+--gpu-memory-utilization 0.7
+
+# Or reduce context length
+--max-model-len 64000
+```
+
+**2. vLLM Multiprocessing Issues**
+The code automatically sets `VLLM_WORKER_MULTIPROC_METHOD=spawn`. If you still encounter issues:
+```bash
+export VLLM_WORKER_MULTIPROC_METHOD=spawn
+```
+
+**3. Evaluation API Errors**
+- Verify API credentials are set correctly
+- Check API endpoint connectivity
+- Monitor rate limits
+- Increase `--nproc` value if rate-limited (up to 32)
+
+**4. Dataset Download Issues**
+The dataset is automatically downloaded from:
+```
+https://opencompass.openxlab.space/utils/VLMEval/MathVision.tsv
+```
+If download fails, manually download and place in `--data-dir`.
+
+**5. Excel Export Errors**
+The code automatically removes illegal Excel characters. If you still encounter issues:
+- Check `clean_for_excel()` function in `run_mathv.py`
+- Ensure `openpyxl` is installed
+
+**6. LaTeX Conversion Errors**
+Install `latex2sympy2` for better LaTeX support:
+```bash
+pip install latex2sympy2
+```
+
+## Advanced Usage
+
+### Custom Image Resolution
+
+Edit `run_mathv.py` to modify image resolution:
+
+```python
+MIN_PIXELS = 768*28*28   # ~0.6M pixels
+MAX_PIXELS = 5120*28*28  # ~4M pixels
+```
+
+### Custom Evaluation Prompts
+
+The evaluation uses in-context examples defined in `eval_utils.py`:
+- Edit `get_gpt4_ICE()` to customize examples
+- Edit `build_mathv_gpt4_prompt()` to modify prompt structure
+
+### Testing with Limited Samples
+
+Use `--num-samples` for quick testing:
+
+```bash
+python run_mathv.py infer \
+    --model-path /path/to/model \
+    --data-dir /path/to/data \
+    --dataset MathVision \
+    --output-file results/test.jsonl \
+    --num-samples 100
+```
+
+### Debugging
+
+Enable debug mode for detailed logs:
+
+```bash
+DEBUG=true python run_mathv.py eval ...
+```
+
+This processes only the first 5 samples in single-threaded mode.
+
+## Citation
+
+If you use this code or the MathVision benchmark, please cite:
+
+```bibtex
+@article{mathvision,
+  title={Measuring Multimodal Mathematical Reasoning with MATH-Vision Dataset}, 
+  author={Ke Wang and Junting Pan and Weikang Shi and Zimu Lu and Mingjie Zhan and Hongsheng Li},
+  journal={arXiv:2402.14804},
+  year={2024}
+}
+```
+
+## License
+
+This code is released under the same license as the Qwen3-VL model.
+
+## Support
+
+For issues and questions:
+- GitHub Issues: [Qwen3-VL Repository](https://github.com/QwenLM/Qwen3-VL)
+- Documentation: See inline code comments and docstrings

evaluation/MathVision/common_utils.py

+import os
+import requests
+import base64
+import hashlib
+import io
+from PIL import Image
+from typing import List, Union
+
+def encode_image_to_base64(image, target_size=None):
+    """Encode an image to base64 string."""
+    if target_size is not None:
+        width, height = image.size
+        # Resize the image while maintaining the aspect ratio
+        if width > height:
+            new_width = target_size
+            new_height = int(height * target_size / width)
+        else:
+            new_height = target_size
+            new_width = int(width * target_size / height)
+        image = image.resize((new_width, new_height))
+    
+    buffer = io.BytesIO()
+    image.save(buffer, format="JPEG")
+    return base64.b64encode(buffer.getvalue()).decode('utf-8')
+
+def decode_base64_to_image(base64_string):
+    """Decode a base64 string to an image."""
+    image_data = base64.b64decode(base64_string)
+    return Image.open(io.BytesIO(image_data))
+
+def decode_base64_to_image_file(base64_string, output_path):
+    """Decode a base64 string and save it to a file."""
+    image = decode_base64_to_image(base64_string)
+    image.save(output_path)
+
+def download_file(url, local_path):
+    """Download a file from a URL to a local path."""
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+    
+    with open(local_path, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+def md5(file_path):
+    """Calculate the MD5 hash of a file."""
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+def toliststr(s):
+    if isinstance(s, str) and (s[0] == '[') and (s[-1] == ']'):
+        return [str(x) for x in eval(s)]
+    elif isinstance(s, str):
+        return [s]
+    elif isinstance(s, list):
+        return [str(x) for x in s]
+    raise NotImplementedError

evaluation/MathVision/dataset_utils.py

+import os
+import pandas as pd
+import numpy as np
+from typing import Dict, Any
+from common_utils import download_file, md5, toliststr, decode_base64_to_image_file
+
+# MathVision dataset URLs and MD5
+MATHVISION_DATASET_URL = {
+    'MathVision': 'https://opencompass.openxlab.space/utils/VLMEval/MathVision.tsv',
+    'MathVision_MINI': 'https://opencompass.openxlab.space/utils/VLMEval/MathVision_MINI.tsv'
+}
+
+MATHVISION_DATASET_MD5 = {
+    'MathVision': '93f6de14f7916e598aa1b7165589831e',
+    'MathVision_MINI': '060fe4fa5d868987ce179307bd5f8a33'
+}
+
+def load_dataset(dataset_name='MathVision'):
+    """Load the MathVision dataset."""
+    data_root = os.path.join(os.environ['LMUData'])
+    os.makedirs(data_root, exist_ok=True)
+    
+    file_name = f"{dataset_name}.tsv"
+    data_path = os.path.join(data_root, file_name)
+    
+    # Download if not exists or MD5 doesn't match
+    if dataset_name in MATHVISION_DATASET_MD5:
+        expected_md5 = MATHVISION_DATASET_MD5[dataset_name]
+        if not os.path.exists(data_path) or md5(data_path) != expected_md5:
+            print(f"Downloading {dataset_name} dataset...")
+            download_file(MATHVISION_DATASET_URL[dataset_name], data_path)
+    
+    # Load the dataset
+    data = pd.read_csv(data_path, sep='\t')
+    
+    # Process the dataset
+    data['index'] = [str(x) for x in data['index']]
+    
+    # Handle image data
+    if 'image' in data:
+        data['image'] = [str(x) for x in data['image']]
+        image_map = {x: y for x, y in zip(data['index'], data['image'])}
+        for k in image_map:
+            if len(image_map[k]) <= 64:
+                idx = image_map[k]
+                assert idx in image_map and len(image_map[idx]) > 64
+                image_map[k] = image_map[idx]
+
+        images = [toliststr(image_map[k]) for k in data['index']]
+        data['image'] = [x[0] if len(x) == 1 else x for x in images]
+    
+    # Handle image paths
+    if 'image_path' in data:
+        paths = [toliststr(x) for x in data['image_path']]
+        data['image_path'] = [x[0] if len(x) == 1 else x for x in paths]
+    
+    # Convert index to int if possible
+    if np.all([isinstance(x, int) or x.isdigit() for x in data['index']]):
+        data['index'] = [int(x) for x in data['index']]
+    
+    return data
+
+def dump_image(line, img_root):
+    """Save image data to disk and return the path."""
+    os.makedirs(img_root, exist_ok=True)
+    
+    if 'image' in line:
+        if isinstance(line['image'], list):
+            tgt_path = []
+            assert 'image_path' in line
+            for img, im_name in zip(line['image'], line['image_path']):
+                path = os.path.join(img_root, im_name)
+                if not os.path.exists(path):
+                    decode_base64_to_image_file(img, path)
+                tgt_path.append(path)
+        else:
+            tgt_path = os.path.join(img_root, f"{line['index']}.jpg")
+            if not os.path.exists(tgt_path):
+                decode_base64_to_image_file(line['image'], tgt_path)
+            tgt_path = [tgt_path]
+    else:
+        assert 'image_path' in line
+        tgt_path = toliststr(line['image_path'])
+    
+    return tgt_path

evaluation/MathVision/eval_instruct.sh

+#!/bin/bash
+
+# MathVision Evaluation Script (Instruct Model)
+# This script evaluates the inference results using GPT-4o
+
+python run_mathv.py eval \
+    --data-dir /path/to/mathvision_data \
+    --input-file results/mathvision_predictions.jsonl \
+    --output-file results/mathvision_eval_results.csv \
+    --dataset MathVision \
+    --eval-model gpt-4o-2024-05-13 \
+    --api-type dash \
+    --nproc 16

evaluation/MathVision/eval_think.sh

+#!/bin/bash
+
+# MathVision Evaluation Script (Thinking Model)
+# This script evaluates the inference results from thinking model using GPT-4o
+
+python run_mathv.py eval \
+    --data-dir /path/to/mathvision_data \
+    --input-file results/mathvision_predictions_thinking.jsonl \
+    --output-file results/mathvision_eval_results_thinking.csv \
+    --dataset MathVision \
+    --eval-model gpt-4o-2024-05-13 \
+    --api-type dash \
+    --nproc 16