mmgenbench.py

import warnings
import pandas as pd
from abc import abstractmethod
from ..smp import *
from .image_base import ImageBaseDataset


class MMGenBench(ImageBaseDataset):

    prompt_list = [
        """
# Role
You are an expert in the field of image understanding, focusing on the \
understanding of images and generating the image caption-prompt.

# Definition Explanation
image caption-prompt: Refers to the caption or description of an image, \
used to provide to a Text-to-Image model to generate a new image.
Text-to-Image model: Can generate a new image based on the provided image \
caption-prompt, such as stable diffusion 3, flux, and other image generation models.

# Task Description
Generate an image caption-prompt based on the input image.

# Key Points and Requirements
1. Accurately understand the input image and precisely generate an image caption-prompt.
2. The generated image caption-prompt, when provided to the Text-to-Image model, requires the \
Text-to-Image model to generate a new image that is as consistent as possible with the input image.
3. The generated image caption-prompt must conform to the preferences of the Text-to-Image model.
4. The generated image caption-prompt should describe the input image in as much \
detail as possible, and it should be between 20 to 60 words.

# Output Format
A string, that is the image caption-prompt. No extra output needed.
"""
    ]
    TYPE = 'GenerateImgPrompt'
    DATASET_URL = {
        'MMGenBench-Test': 'https://huggingface.co/datasets/lerogo/MMGenBench/resolve/main/MMGenBench-Test.tsv',
        'MMGenBench-Domain': 'https://huggingface.co/datasets/lerogo/MMGenBench/resolve/main/MMGenBench-Domain.tsv',
    }
    PROMPT_MAP = {
        'MMGenBench-Test': prompt_list[0],
        'MMGenBench-Domain': prompt_list[0],
    }
    DATASET_MD5 = {
        'MMGenBench-Test': "94f8dac6bbf7c20be403f99adeaa73da",
        'MMGenBench-Domain': "5c10daf6e2c5f08bdfb0701aa6db86bb",
    }

    def __init__(self, dataset='MMGenBench', **kwargs):
        super().__init__(dataset, **kwargs)
        warnings.warn('This dataset is for inference only and does not support direct output of evaluation results.\n')
        warnings.warn('Please refer to "https://github.com/lerogo/MMGenBench" for more evaluation information.\n')

    def load_data(self, dataset):
        data = super().load_data(dataset)
        if 'question' not in data:
            data['question'] = [(
                self.PROMPT_MAP[dataset]
            )] * len(data)
        return data

    # Given the prediction file, return the evaluation results in the format of a dictionary or pandas dataframe
    @abstractmethod
    def evaluate(self, eval_file, **judge_kwargs):
        warnings.warn('This evaluation method is not supported.\n')
        warnings.warn('Please refer to "https://github.com/lerogo/MMGenBench" for more evaluation information.\n')
        return None