Skip to content

GitLab

"tests/models/gpt2/test_tokenization_gpt2.py" did not exist on "00aa9dbca29dcf0e3a624354ef5c80a8e5226339"
Commit 26e59280 authored by wanglch's avatar wanglch
Browse files

Initial commit

parents
Pipeline #2674 failed with stages
in 0 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 3316 additions and 0 deletions
internvl_chat/eval/mantis_eval/README.md 0 → 100644
View file @ 26e59280
# README for Evaluation
## 🌟 Overview
This script provides an evaluation pipeline for `Mantis-Eval`.
## 🗂️ Data Preparation
Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
### Mantis-Eval
The evaluation script will automatically download the Mantis Eval dataset from HuggingFace, and the cached path is `data/mantis_eval`.
## 🏃 Evaluation Execution
> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
To run the evaluation, execute the following command on an 8-GPU setup:
```shell
torchrun --nproc_per_node=8 eval/mantis_eval/evaluate_mantis.py --checkpoint ${CHECKPOINT} --dynamic
```
Alternatively, you can run the following simplified command:
```shell
GPUS=8 sh evaluate.sh ${CHECKPOINT} mantis --dynamic
```
### Arguments
The following arguments can be configured for the evaluation script:
| Argument | Type | Default | Description |
| ---------------- | ------ | --------------- | ----------------------------------------------------------------------------------------------------------------- |
| `--checkpoint` | `str` | `''` | Path to the model checkpoint. |
| `--datasets` | `str` | `'Mantis-Eval'` | Comma-separated list of datasets to evaluate. |
| `--dynamic` | `flag` | `False` | Enables dynamic high resolution preprocessing. |
| `--max-num` | `int` | `6` | Maximum tile number for dynamic high resolution. |
| `--load-in-8bit` | `flag` | `False` | Loads the model weights in 8-bit precision. |
| `--auto` | `flag` | `False` | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |
internvl_chat/eval/mantis_eval/evaluate_mantis.py 0 → 100644
View file @ 26e59280
import argparse
import itertools
import json
import os
import random
import time
from functools import partial
import numpy as np
import torch
from datasets import load_dataset
from internvl.model import load_model_and_tokenizer
from internvl.train.dataset import build_transform, dynamic_preprocess
from tqdm import tqdm
ds_collections = {
'Mantis-Eval': {
'root': 'TIGER-Lab/Mantis-Eval',
'max_new_tokens': 50,
'min_new_tokens': 1,
'split': 'test'
},
}
def collate_fn(batches, tokenizer):
pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
questions = [_['question'] for _ in batches]
answers = [_['answer'] for _ in batches]
num_patches_lists = [_['num_patches_list'] for _ in batches]
options = [_['option'] for _ in batches]
lines = [_['line'] for _ in batches]
return pixel_values, questions, answers, num_patches_lists, options, lines
class MantisEvalDataset(torch.utils.data.Dataset):
def __init__(self, root, split, prompt, input_size=224, dynamic_image_size=False,
use_thumbnail=False, max_num=6):
dataset = load_dataset(root, split=split, cache_dir=os.path.join(os.getcwd(), 'data/mantis_eval/'))
self.data = dataset
self.prompt = prompt
self.input_size = input_size
self.dynamic_image_size = dynamic_image_size
self.use_thumbnail = use_thumbnail
self.max_num = max_num
self.transform = build_transform(is_train=False, input_size=input_size)
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
data = self.data[idx]
question_type = data['question_type']
question = data['question']
image_list = data['images']
del data['images']
del data['data_source']
images_to_remove = ' '.join(['<image>'] * len(image_list))
question = question.replace(images_to_remove, '').strip()
for i in range(len(image_list)):
question = question.replace('<image>', f'Image-{i + 1}', 1)
options = data['options']
options = [item.strip() for item in options]
answer = data['answer']
choice_txt = '\n'.join(options)
num_patches_list = []
if self.dynamic_image_size:
images = []
for image in image_list:
tiles = dynamic_preprocess(image, image_size=self.input_size,
use_thumbnail=self.use_thumbnail,
max_num=max(1, self.max_num // len(image_list)))
images += tiles
num_patches_list.append(len(tiles))
else:
images = image_list
num_patches_list.append(1)
pixel_values = [self.transform(image) for image in images]
pixel_values = torch.stack(pixel_values)
if question_type == 'multi-choice':
question = question + '\n' + choice_txt + '\n' + self.prompt[question_type]
else:
question = question + '\n' + self.prompt[question_type]
question = question.strip()
if len(image_list) == 1:
prefix = '<image>\n'
else:
prefix = ''.join([f'Image-{i + 1}: <image>\n' for i in range(len(image_list))])
question = prefix + question
return {
'question': question,
'pixel_values': pixel_values,
'answer': answer,
'option': options,
'num_patches_list': num_patches_list,
'line': data
}
class InferenceSampler(torch.utils.data.sampler.Sampler):
def __init__(self, size):
self._size = int(size)
assert size > 0
self._rank = torch.distributed.get_rank()
self._world_size = torch.distributed.get_world_size()
self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
@staticmethod
def _get_local_indices(total_size, world_size, rank):
shard_size = total_size // world_size
left = total_size % world_size
shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
begin = sum(shard_sizes[:rank])
end = min(sum(shard_sizes[:rank + 1]), total_size)
return range(begin, end)
def __iter__(self):
yield from self._local_indices
def __len__(self):
return len(self._local_indices)
def evaluate_chat_model():
prompt = {
'multi-choice': "Answer with the option's letter from the given choices directly.",
'short-answer': 'Answer the question using a single word or phrase.'
}
random.seed(args.seed)
for ds_name in args.datasets:
dataset = MantisEvalDataset(
root=ds_collections[ds_name]['root'],
split=ds_collections[ds_name]['split'],
prompt=prompt,
input_size=image_size,
dynamic_image_size=args.dynamic,
use_thumbnail=use_thumbnail,
max_num=args.max_num
)
dataloader = torch.utils.data.DataLoader(
dataset=dataset,
sampler=InferenceSampler(len(dataset)),
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
drop_last=False,
collate_fn=partial(collate_fn, tokenizer=tokenizer),
)
outputs = []
for _, (pixel_values, questions, answers, num_patches_lists, options, lines) in tqdm(enumerate(dataloader)):
pixel_values = pixel_values.to(torch.bfloat16).cuda()
generation_config = dict(
num_beams=args.num_beams,
max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
do_sample=True if args.temperature > 0 else False,
temperature=args.temperature,
)
pred = model.chat(
tokenizer=tokenizer,
pixel_values=pixel_values,
question=questions[0],
generation_config=generation_config,
num_patches_list=num_patches_lists[0],
verbose=True
)
preds = [pred]
for question, pred, answer, line in zip(questions, preds, answers, lines):
line['question'] = question
line['pred'] = pred
line['answer'] = answer
options = line['options']
question_type = line['question_type']
if question_type == 'multi-choice':
if len(pred) == 3 and pred[0] == '(' and pred[-1] == ')':
pred = pred[1:-1]
if pred == options[ord(answer) - ord('A')] or pred == answer:
line['correct'] = 1
else:
line['correct'] = 0
else:
if pred.lower() == answer.lower():
line['correct'] = 1
else:
line['correct'] = 0
outputs.append(line)
torch.distributed.barrier()
world_size = torch.distributed.get_world_size()
merged_outputs = [None for _ in range(world_size)]
torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
merged_outputs = [json.loads(_) for _ in merged_outputs]
merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
if torch.distributed.get_rank() == 0:
print(f'Evaluating {ds_name} ...')
time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
results_file = f'{ds_name}_{time_prefix}.jsonl'
output_path = os.path.join(args.out_dir, results_file)
writer = open(output_path, 'w')
for item in merged_outputs:
writer.write(json.dumps(item) + '\n')
writer.close()
print('Results saved to {}'.format(output_path))
multi_choice_items = [item for item in merged_outputs if item['question_type'] == 'multi-choice']
if len(multi_choice_items) > 0:
print(f'Multi-choice Accuracy: {np.mean([q["correct"] for q in multi_choice_items]):.4f}')
open_ended_items = [item for item in merged_outputs if item['question_type'] == 'short-answer']
if len(open_ended_items) > 0:
print(f'Short-answer Accuracy: {np.mean([q["correct"] for q in open_ended_items]):.4f}')
if len(merged_outputs) > 0:
print(f"Overall Accuracy: {np.mean([q['correct'] for q in merged_outputs]):.4f}")
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--checkpoint', type=str, default='')
parser.add_argument('--datasets', type=str, default='Mantis-Eval')
parser.add_argument('--batch-size', type=int, default=1)
parser.add_argument('--num-workers', type=int, default=1)
parser.add_argument('--num-beams', type=int, default=1)
parser.add_argument('--temperature', type=float, default=0.0)
parser.add_argument('--out-dir', type=str, default='results')
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--dynamic', action='store_true')
parser.add_argument('--max-num', type=int, default=6)
parser.add_argument('--load-in-8bit', action='store_true')
parser.add_argument('--load-in-4bit', action='store_true')
parser.add_argument('--auto', action='store_true')
args = parser.parse_args()
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir, exist_ok=True)
args.datasets = args.datasets.split(',')
print('datasets:', args.datasets)
assert args.batch_size == 1, 'Only batch size 1 is supported'
torch.distributed.init_process_group(
backend='nccl',
world_size=int(os.getenv('WORLD_SIZE', '1')),
rank=int(os.getenv('RANK', '0')),
)
torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
model, tokenizer = load_model_and_tokenizer(args)
image_size = model.config.force_image_size or model.config.vision_config.image_size
use_thumbnail = model.config.use_thumbnail
total_params = sum(p.numel() for p in model.parameters()) / 1e9
if total_params > 20 or args.dynamic:
args.num_beams = 1
print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
else:
print(f'[test] total_params: {total_params}B')
print(f'[test] image_size: {image_size}')
print(f'[test] template: {model.config.template}')
print(f'[test] dynamic_image_size: {args.dynamic}')
print(f'[test] use_thumbnail: {use_thumbnail}')
print(f'[test] max_num: {args.max_num}')
evaluate_chat_model()
internvl_chat/eval/mathvista/README.md 0 → 100644
View file @ 26e59280
# README for Evaluation
## 🌟 Overview
This script provides an evaluation pipeline for `MathVista`.
For scoring, we use **GPT-4-0613** as the evaluation model.
While the provided code can run the benchmark, we recommend using [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) for testing this benchmark if you aim to align results with our technical report.
## 🗂️ Data Preparation
Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
### MathVista
Follow the instructions below to prepare the data：
```bash
# Step 1: Create the data directory
mkdir -p data/MathVista && cd data/MathVista
# Step 2: Download the annotation
wget https://huggingface.co/datasets/AI4Math/MathVista/raw/main/annot_testmini.json
cd ../..
```
After preparation is complete, the directory structure is:
```
MathVista
└── annot_testmini.json
```
## 🏃 Evaluation Execution
> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
To run the evaluation, execute the following command on an 8-GPU setup:
```shell
export OPENAI_API_KEY="your_openai_api_key"
# Test the testmini set
torchrun --nproc_per_node=8 eval/mathvista/evaluate_mathvista.py --checkpoint ${CHECKPOINT} --dynamic --datasets MathVista_testmini
# Test the test set
torchrun --nproc_per_node=8 eval/mathvista/evaluate_mathvista.py --checkpoint ${CHECKPOINT} --dynamic --datasets MathVista_test
```
Alternatively, you can run the following simplified command:
```shell
export OPENAI_API_KEY="your_openai_api_key"
# Test the testmini set
GPUS=8 sh evaluate.sh ${CHECKPOINT} mathvista-testmini --dynamic
# Test the test set
GPUS=8 sh evaluate.sh ${CHECKPOINT} mathvista-test --dynamic
```
### Arguments
The following arguments can be configured for the evaluation script:
| Argument | Type | Default | Description |
| ---------------- | ------ | ---------------------- | ----------------------------------------------------------------------------------------------------------------- |
| `--checkpoint` | `str` | `''` | Path to the model checkpoint. |
| `--datasets` | `str` | `'MathVista_testmini'` | Comma-separated list of datasets to evaluate. |
| `--dynamic` | `flag` | `False` | Enables dynamic high resolution preprocessing. |
| `--max-num` | `int` | `6` | Maximum tile number for dynamic high resolution. |
| `--load-in-8bit` | `flag` | `False` | Loads the model weights in 8-bit precision. |
| `--auto` | `flag` | `False` | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |
internvl_chat/eval/mathvista/calculate_score.py 0 → 100644
View file @ 26e59280
import argparse
import pandas as pd
# !pip install python-Levenshtein
from Levenshtein import distance
from utilities import *
def get_most_similar(prediction, choices):
"""
Use the Levenshtein distance (or edit distance) to determine which of the choices is most similar to the given prediction
"""
distances = [distance(prediction, choice) for choice in choices]
ind = distances.index(min(distances))
return choices[ind]
# return min(choices, key=lambda choice: distance(prediction, choice))
def normalize_extracted_answer(extraction, choices, question_type, answer_type, precision):
"""
Normalize the extracted answer to match the answer type
"""
if question_type == 'multi_choice':
# make sure the extraction is a string
if isinstance(extraction, str):
extraction = extraction.strip()
else:
try:
extraction = str(extraction)
except:
extraction = ''
# extract "A" from "(A) text"
letter = re.findall(r'\(([a-zA-Z])\)', extraction)
if len(letter) > 0:
extraction = letter[0].upper()
options = [chr(ord('A') + i) for i in range(len(choices))]
if extraction in options:
# convert option letter to text, e.g. "A" -> "text"
ind = options.index(extraction)
extraction = choices[ind]
else:
# select the most similar option
extraction = get_most_similar(extraction, choices)
assert extraction in choices
elif answer_type == 'integer':
try:
extraction = str(int(float(extraction)))
except:
extraction = None
elif answer_type == 'float':
try:
extraction = str(round(float(extraction), int(precision)))
except:
extraction = None
elif answer_type == 'list':
try:
extraction = str(extraction)
except:
extraction = None
return extraction
def safe_equal(prediction, answer):
"""
Check if the prediction is equal to the answer, even if they are of different types
"""
try:
if prediction == answer:
return True
return False
except Exception as e:
print(e)
return False
def get_acc_with_contion(res_pd, key, value):
if key == 'skills':
# if value in res_pd[key]:
total_pd = res_pd[res_pd[key].apply(lambda x: value in x)]
else:
total_pd = res_pd[res_pd[key] == value]
correct_pd = total_pd[total_pd['true_false'] == True] # noqa: E712
acc = '{:.2f}'.format(len(correct_pd) / len(total_pd) * 100)
return len(correct_pd), len(total_pd), acc
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--output_dir', type=str, default='./results')
parser.add_argument('--output_file', type=str, default='output.json')
parser.add_argument('--score_file', type=str, default='scores.json')
parser.add_argument('--gt_file', type=str, default='./data/MathVista/annot_testmini.json', help='ground truth file')
parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
parser.add_argument('--rerun', action='store_true', help='rerun the evaluation')
parser.add_argument('--caculate_gain', action='store_true', help='caculate the score gains over random guess')
parser.add_argument('--random_file', type=str, default='score_random_guess.json')
args = parser.parse_args()
# args
output_file = os.path.join(args.output_dir, args.output_file)
# # quick test
# output_file = '../results/llava-llama-2-13b/output_llava_llama_2_13b.json'
# read json
print(f'Reading {output_file}...')
results = read_json(output_file)
# read ground truth
print(f'Reading {args.gt_file}...')
gts = read_json(args.gt_file)
# full pids
full_pids = list(results.keys())
if args.number > 0:
full_pids = full_pids[:min(args.number, len(full_pids))]
print('Number of testing problems:', len(full_pids))
## [1] Evaluate if the prediction is true or false
print('\nEvaluating the predictions...')
update_json_flag = False
for pid in full_pids:
problem = results[pid]
# print(problem)
if args.rerun:
if 'prediction' in problem:
del problem['prediction']
if 'true_false' in problem:
del problem['true_false']
choices = problem['choices']
question_type = problem['question_type']
answer_type = problem['answer_type']
precision = problem['precision']
extraction = problem['extraction']
if 'answer' in problem:
answer = problem['answer']
else:
if pid in gts:
answer = gts[pid]['answer']
else:
answer = ''
problem['answer'] = answer
# normalize the extracted answer to match the answer type
prediction = normalize_extracted_answer(extraction, choices, question_type, answer_type, precision)
# verify the prediction is true or false
true_false = safe_equal(prediction, answer)
# update the problem
if 'true_false' not in problem:
update_json_flag = True
elif true_false != problem['true_false']:
update_json_flag = True
if 'prediction' not in problem:
update_json_flag = True
elif prediction != problem['prediction']:
update_json_flag = True
problem['prediction'] = prediction
problem['true_false'] = true_false
# save the updated json
if update_json_flag:
print('\n!!!Some problems are updated.!!!')
print(f'\nSaving {output_file}...')
save_json(results, output_file)
## [2] Calculate the average accuracy
total = len(full_pids)
correct = 0
for pid in full_pids:
if results[pid]['true_false']:
correct += 1
accuracy = str(round(correct / total * 100, 2))
print(f'\nCorrect: {correct}, Total: {total}, Accuracy: {accuracy}%')
scores = {'average': {'accuracy': accuracy, 'correct': correct, 'total': total}}
## [3] Calculate the fine-grained accuracy scores
# merge the 'metadata' attribute into the data
for pid in results:
results[pid].update(results[pid].pop('metadata'))
# convert the data to a pandas DataFrame
df = pd.DataFrame(results).T
print(len(df))
print('Number of test problems:', len(df))
# assert len(df) == 1000 # Important!!!
# asign the target keys for evaluation
target_keys = ['question_type', 'answer_type', 'language', 'source', 'category', 'task', 'context', 'grade',
'skills']
for key in target_keys:
print(f'\nType: [{key}]')
# get the unique values of the key
if key == 'skills':
# the value is a list
values = []
for i in range(len(df)):
values += df[key][i]
values = list(set(values))
else:
values = df[key].unique()
# print(values)
# calculate the accuracy for each value
scores[key] = {}
for value in values:
correct, total, acc = get_acc_with_contion(df, key, value)
if total > 0:
print(f'[{value}]: {acc}% ({correct}/{total})')
scores[key][value] = {'accuracy': acc, 'correct': correct, 'total': total}
# sort the scores by accuracy
scores[key] = dict(sorted(scores[key].items(), key=lambda item: float(item[1]['accuracy']), reverse=True))
# save the scores
scores_file = os.path.join(args.output_dir, args.score_file)
print(f'\nSaving {scores_file}...')
save_json(scores, scores_file)
print('\nDone!')
# [4] Calculate the score gains over random guess
if args.caculate_gain:
random_file = os.path.join(args.output_dir, args.random_file)
random_scores = json.load(open(random_file))
print('\nCalculating the score gains...')
for key in scores:
if key == 'average':
gain = round(float(scores[key]['accuracy']) - float(random_scores[key]['accuracy']), 2)
scores[key]['acc_gain'] = gain
else:
for sub_key in scores[key]:
gain = round(
float(scores[key][sub_key]['accuracy']) - float(random_scores[key][sub_key]['accuracy']), 2)
scores[key][sub_key]['acc_gain'] = str(gain)
# save the score gains
print(f'\nSaving {scores_file}...')
save_json(scores, scores_file)
print('\nDone!')
internvl_chat/eval/mathvista/evaluate_mathvista.py 0 → 100644
View file @ 26e59280
import argparse
import itertools
import json
import os
import random
import time
from functools import partial
import torch
from datasets import concatenate_datasets, load_dataset
from internvl.model import load_model_and_tokenizer
from internvl.train.dataset import build_transform, dynamic_preprocess
from tqdm import tqdm
ds_collections = {
'MathVista_testmini': {
'root': 'AI4Math/MathVista',
'max_new_tokens': 4096,
'min_new_tokens': 1,
'split': 'testmini'
},
'MathVista_test': {
'root': 'AI4Math/MathVista',
'max_new_tokens': 4096,
'min_new_tokens': 1,
'split': 'test'
},
}
COT_INSTRUCTION = (
'Your task is to answer the question below. '
"Give step by step reasoning before you answer, and when you're ready to answer, "
"please use the format \"Final answer: ..\""
'\n\n'
'Question:'
'\n\n'
'{question}'
)
def collate_fn(batches, tokenizer):
pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
data_items = [_['data_item'] for _ in batches]
return pixel_values, data_items
class MathVistaDataset(torch.utils.data.Dataset):
def __init__(self, root, split, input_size=224, dynamic_image_size=False,
use_thumbnail=False, max_num=6):
dataset = load_dataset(root, cache_dir=os.path.join(os.getcwd(), 'data/MathVista/'))
self.data = dataset[split]
self.input_size = input_size
self.dynamic_image_size = dynamic_image_size
self.use_thumbnail = use_thumbnail
self.max_num = max_num
self.transform = build_transform(is_train=False, input_size=input_size)
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
data_item = self.data[idx]
image = data_item['decoded_image']
del data_item['decoded_image']
if self.dynamic_image_size:
images = dynamic_preprocess(image, image_size=self.input_size,
use_thumbnail=self.use_thumbnail,
max_num=self.max_num)
else:
images = [image]
pixel_values = [self.transform(image) for image in images]
pixel_values = torch.stack(pixel_values)
return {
'pixel_values': pixel_values,
'data_item': data_item,
}
class InferenceSampler(torch.utils.data.sampler.Sampler):
def __init__(self, size):
self._size = int(size)
assert size > 0
self._rank = torch.distributed.get_rank()
self._world_size = torch.distributed.get_world_size()
self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
@staticmethod
def _get_local_indices(total_size, world_size, rank):
shard_size = total_size // world_size
left = total_size % world_size
shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
begin = sum(shard_sizes[:rank])
end = min(sum(shard_sizes[:rank + 1]), total_size)
return range(begin, end)
def __iter__(self):
yield from self._local_indices
def __len__(self):
return len(self._local_indices)
def evaluate_chat_model():
random.seed(args.seed)
for ds_name in args.datasets:
dataset = MathVistaDataset(
root=ds_collections[ds_name]['root'],
split=ds_collections[ds_name]['split'],
input_size=image_size,
dynamic_image_size=args.dynamic,
use_thumbnail=use_thumbnail,
max_num=args.max_num
)
dataloader = torch.utils.data.DataLoader(
dataset=dataset,
sampler=InferenceSampler(len(dataset)),
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
drop_last=False,
collate_fn=partial(collate_fn, tokenizer=tokenizer),
)
outputs = []
for _, (pixel_values, data_items) in tqdm(enumerate(dataloader)):
if args.cot:
question = COT_INSTRUCTION.format(question=data_items[0]['query'])
else:
question = data_items[0]['query']
pixel_values = pixel_values.to(torch.bfloat16).cuda()
generation_config = dict(
num_beams=args.num_beams,
max_new_tokens=ds_collections[ds_name]['max_new_tokens'] if not args.cot else 4096,
min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
do_sample=True if args.temperature > 0 else False,
temperature=args.temperature,
)
pred = model.chat(
tokenizer=tokenizer,
pixel_values=pixel_values,
question=question,
generation_config=generation_config,
verbose=True
)
data_item = data_items[0]
data_item['response'] = pred
outputs.append(data_item)
torch.distributed.barrier()
world_size = torch.distributed.get_world_size()
merged_outputs = [None for _ in range(world_size)]
torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
merged_outputs = [json.loads(_) for _ in merged_outputs]
merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
if torch.distributed.get_rank() == 0:
temp = {}
for data_item in merged_outputs:
pid = data_item['pid']
temp[pid] = data_item
print(f'Evaluating {ds_name} ...')
time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
results_file = f'{ds_name}_{time_prefix}.json'
output_path = os.path.join(args.out_dir, results_file)
json.dump(temp, open(output_path, 'w'), indent=4)
print('Results saved to {}'.format(output_path))
if args.cot:
cmd = f'python eval/mathvista/extract_answer.py --output_file {results_file} --output_dir {args.out_dir} --quick_extract'
else:
cmd = f'python eval/mathvista/extract_answer.py --output_file {results_file} --output_dir {args.out_dir}'
print(cmd)
os.system(cmd)
cmd = f'python eval/mathvista/calculate_score.py --output_file {results_file} --output_dir {args.out_dir} --score_file {results_file[:-5]}_score.json'
print(cmd)
os.system(cmd)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--checkpoint', type=str, default='')
parser.add_argument('--datasets', type=str, default='MathVista_testmini')
parser.add_argument('--batch-size', type=int, default=1)
parser.add_argument('--num-workers', type=int, default=1)
parser.add_argument('--num-beams', type=int, default=1)
parser.add_argument('--temperature', type=float, default=0.0)
parser.add_argument('--out-dir', type=str, default='results')
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--dynamic', action='store_true')
parser.add_argument('--max-num', type=int, default=6)
parser.add_argument('--load-in-8bit', action='store_true')
parser.add_argument('--load-in-4bit', action='store_true')
parser.add_argument('--auto', action='store_true')
parser.add_argument('--cot', action='store_true')
args = parser.parse_args()
model_name = '_'.join(args.checkpoint.split('/')[-2:])
model_name = f'{model_name}_cot' if args.cot else model_name
args.out_dir = os.path.join(args.out_dir, model_name)
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir, exist_ok=True)
args.datasets = args.datasets.split(',')
print('datasets:', args.datasets)
assert args.batch_size == 1, 'Only batch size 1 is supported'
torch.distributed.init_process_group(
backend='nccl',
world_size=int(os.getenv('WORLD_SIZE', '1')),
rank=int(os.getenv('RANK', '0')),
)
torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
model, tokenizer = load_model_and_tokenizer(args)
image_size = model.config.force_image_size or model.config.vision_config.image_size
use_thumbnail = model.config.use_thumbnail
total_params = sum(p.numel() for p in model.parameters()) / 1e9
if total_params > 20 or args.dynamic:
args.num_beams = 1
print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
else:
print(f'[test] total_params: {total_params}B')
print(f'[test] image_size: {image_size}')
print(f'[test] template: {model.config.template}')
print(f'[test] dynamic_image_size: {args.dynamic}')
print(f'[test] use_thumbnail: {use_thumbnail}')
print(f'[test] max_num: {args.max_num}')
evaluate_chat_model()
internvl_chat/eval/mathvista/extract_answer.py 0 → 100644
View file @ 26e59280
import argparse
from tqdm import tqdm
from utilities import *
api_key = os.getenv('OPENAI_API_KEY')
print(api_key)
# proxy_url = ""
# proxies = {
# "http://": f"{proxy_url}",
# "https://": f"{proxy_url}",
# }
# http_client = httpx.Client(proxies=proxies)
http_client = None
# load demo prompt
from prompts.ext_ans import demo_prompt
def verify_extraction(extraction):
extraction = extraction.strip()
if extraction == '' or extraction is None:
return False
return True
def create_test_prompt(demo_prompt, query, response):
demo_prompt = demo_prompt.strip()
test_prompt = f'{query}\n\n{response}'
full_prompt = f'{demo_prompt}\n\n{test_prompt}\n\nExtracted answer: '
return full_prompt
def _extract_answer(text):
match = re.search(r'(Final answer:|Answer:)\s*(.*)', text, re.IGNORECASE)
if match:
return match.group(2).strip()
return text
def extract_answer(response, problem, quick_extract=False):
question_type = problem['question_type']
answer_type = problem['answer_type']
choices = problem['choices']
query = problem['query']
if response == '':
return ''
if question_type == 'multi_choice' and response in choices:
return response
if answer_type == 'integer':
try:
extraction = int(response)
return str(extraction)
except:
pass
if answer_type == 'float':
try:
extraction = str(float(response))
return extraction
except:
pass
# quick extraction
if quick_extract:
print('Quickly extracting answer...')
# The answer is "text". -> "text"
try:
result = _extract_answer(response)
return result
# result = re.search(r'The answer is "(.*)"\.', response)
# if result:
# extraction = result.group(1)
# return extraction
except:
pass
# general extraction
try:
full_prompt = create_test_prompt(demo_prompt, query, response)
extraction = get_chat_response(full_prompt, api_key, patience=5, http_client=http_client)
return extraction
except Exception as e:
print(e)
print(f'Error in extracting answer for {pid}')
return ''
if __name__ == '__main__':
parser = argparse.ArgumentParser()
# input
parser.add_argument('--output_dir', type=str, default='./results')
parser.add_argument('--output_file', type=str, default='mathvista_answer.json')
parser.add_argument('--response_label', type=str, default='response', help='response label for the input file')
# model
parser.add_argument('--llm_engine', type=str, default='gpt-4-0613', help='llm engine',
choices=['gpt-3.5-turbo', 'gpt-3.5', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613'])
parser.add_argument('--number', type=int, default=-1, help='number of problems to run')
parser.add_argument('--quick_extract', action='store_true', help='use rules to extract answer for some problems')
parser.add_argument('--rerun', action='store_true', help='rerun the answer extraction')
# output
parser.add_argument('--save_every', type=int, default=10, help='save every n problems')
parser.add_argument('--output_label', type=str, default='', help='label for the output file')
args = parser.parse_args()
# args
label = args.response_label
result_file = os.path.join(args.output_dir, args.output_file)
if args.output_label != '':
output_file = result_file.replace('.json', f'_{args.output_label}.json')
else:
output_file = result_file
# read results
print(f'Reading {result_file}...')
results = read_json(result_file)
# full pids
full_pids = list(results.keys())
if args.number > 0:
full_pids = full_pids[:min(args.number, len(full_pids))]
print('Number of testing problems:', len(full_pids))
# test pids
if args.rerun:
test_pids = full_pids
else:
test_pids = []
for pid in full_pids:
# print(pid)
if 'extraction' not in results[pid] or not verify_extraction(results[pid]['extraction']):
test_pids.append(pid)
test_num = len(test_pids)
print('Number of problems to run:', test_num)
# print(test_pids)
# tqdm, enumerate results
for i, pid in enumerate(tqdm(test_pids)):
problem = results[pid]
assert label in problem
response = problem[label]
extraction = extract_answer(response, problem, args.quick_extract)
results[pid]['extraction'] = extraction
if i % args.save_every == 0 or i == test_num - 1:
print(f'Saving results to {output_file}...')
save_json(results, output_file)
print(f'Results saved.')
internvl_chat/eval/mathvista/prompts/ext_ans.py 0 → 100644
View file @ 26e59280
# pids = 852, 104, 824, 506, 540
demo_prompt = """
Please read the following example. Then extract the answer from the model response and type it at the end of the prompt.
Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end.
Question: Which number is missing?
Model response: The number missing in the sequence is 14.
Extracted answer: 14
Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end.
Question: What is the fraction of females facing the camera?
Model response: The fraction of females facing the camera is 0.6, which means that six out of ten females in the group are facing the camera.
Extracted answer: 0.6
Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end.
Question: How much money does Luca need to buy a sour apple candy and a butterscotch candy? (Unit: $)
Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy.
Extracted answer: 1.45
Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end.
Question: Between which two years does the line graph saw its maximum peak?
Model response: The line graph saw its maximum peak between 2007 and 2008.
Extracted answer: [2007, 2008]
Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end.
Question: What fraction of the shape is blue?\nChoices:\n(A) 3/11\n(B) 8/11\n(C) 6/11\n(D) 3/5
Model response: The correct answer is (B) 8/11.
Extracted answer: B
"""
internvl_chat/eval/mathvista/utilities.py 0 → 100644
View file @ 26e59280
import json
import os
import pickle
import re
import time
import cv2
import openai
from word2number import w2n
openai_client = None
def create_dir(output_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
def read_csv(file):
data = []
with open(file, 'r') as f:
for line in f:
data.append(line.strip())
return data
def read_pandas_csv(csv_path):
# read a pandas csv sheet
import pandas as pd
df = pd.read_csv(csv_path)
return df
def read_json(path):
with open(path, 'r', encoding='utf-8') as f:
return json.load(f)
def read_jsonl(file):
with open(file, 'r') as f:
data = [json.loads(line) for line in f]
return data
def read_pickle(path):
with open(path, 'rb') as f:
return pickle.load(f)
def save_json(data, path):
with open(path, 'w') as f:
json.dump(data, f, indent=4)
def save_array_img(path, image):
cv2.imwrite(path, image)
def contains_digit(text):
# check if text contains a digit
if any(char.isdigit() for char in text):
return True
return False
def contains_number_word(text):
# check if text contains a number word
ignore_words = ['a', 'an', 'point']
words = re.findall(r'\b\w+\b', text) # This regex pattern matches any word in the text
for word in words:
if word in ignore_words:
continue
try:
w2n.word_to_num(word)
return True # If the word can be converted to a number, return True
except ValueError:
continue # If the word can't be converted to a number, continue with the next word
# check if text contains a digit
if any(char.isdigit() for char in text):
return True
return False # If none of the words could be converted to a number, return False
def contains_quantity_word(text, special_keep_words=[]):
# check if text contains a quantity word
quantity_words = ['most', 'least', 'fewest'
'more', 'less', 'fewer',
'largest', 'smallest', 'greatest',
'larger', 'smaller', 'greater',
'highest', 'lowest', 'higher', 'lower',
'increase', 'decrease',
'minimum', 'maximum', 'max', 'min',
'mean', 'average', 'median',
'total', 'sum', 'add', 'subtract',
'difference', 'quotient', 'gap',
'half', 'double', 'twice', 'triple',
'square', 'cube', 'root',
'approximate', 'approximation',
'triangle', 'rectangle', 'circle', 'square', 'cube', 'sphere', 'cylinder', 'cone', 'pyramid',
'multiply', 'divide',
'percentage', 'percent', 'ratio', 'proportion', 'fraction', 'rate',
]
quantity_words += special_keep_words # dataset specific words
words = re.findall(r'\b\w+\b', text) # This regex pattern matches any word in the text
if any(word in quantity_words for word in words):
return True
return False # If none of the words could be converted to a number, return False
def is_bool_word(text):
if text in ['Yes', 'No', 'True', 'False',
'yes', 'no', 'true', 'false',
'YES', 'NO', 'TRUE', 'FALSE']:
return True
return False
def is_digit_string(text):
# remove ".0000"
text = text.strip()
text = re.sub(r'\.0+$', '', text)
try:
int(text)
return True
except ValueError:
return False
def is_float_string(text):
# text is a float string if it contains a "." and can be converted to a float
if '.' in text:
try:
float(text)
return True
except ValueError:
return False
return False
def copy_image(image_path, output_image_path):
from shutil import copyfile
copyfile(image_path, output_image_path)
def copy_dir(src_dir, dst_dir):
from shutil import copytree
# copy the source directory to the target directory
copytree(src_dir, dst_dir)
import PIL.Image as Image
def get_image_size(img_path):
img = Image.open(img_path)
width, height = img.size
return width, height
def get_chat_response(promot, api_key, model='gpt-3.5-turbo', temperature=0, max_tokens=256, n=1, patience=10000000,
sleep_time=0, http_client=None):
global openai_client
if openai_client is None:
print(f'{api_key=}')
openai_client = openai.OpenAI(api_key=api_key, http_client=http_client)
messages = [
{'role': 'user', 'content': promot},
]
while patience > 0:
patience -= 1
try:
response = openai_client.chat.completions.create(
model=model,
messages=messages,
# api_key=api_key,
temperature=temperature,
max_tokens=max_tokens,
n=n,
)
response = response.to_dict()
if n == 1:
prediction = response['choices'][0]['message']['content'].strip()
if prediction != '' and prediction is not None:
return prediction
else:
prediction = [choice['message']['content'].strip() for choice in response['choices']]
if prediction[0] != '' and prediction[0] is not None:
return prediction
except Exception as e:
if 'Rate limit' not in str(e):
print(e)
if 'Please reduce the length of the messages' in str(e):
print('!!Reduce promot size')
# reduce input prompt and keep the tail
new_size = int(len(promot) * 0.9)
new_start = len(promot) - new_size
promot = promot[new_start:]
messages = [
{'role': 'user', 'content': promot},
]
if sleep_time > 0:
time.sleep(sleep_time)
return ''
internvl_chat/eval/mirb/README.md 0 → 100644
View file @ 26e59280
# README for Evaluation
## 🌟 Overview
This script provides an evaluation pipeline for `MIRB`.
## 🗂️ Data Preparation
Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
### MIRB
Follow the instructions below to prepare the data:
```shell
# Step 1: Download annotation files
cd data/
GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datasets/VLLMs/MIRB
# Step 2: Download and unzip the image files
cd MIRB/ && rm -rf images.zip
wget https://huggingface.co/datasets/VLLMs/MIRB/resolve/main/images.zip
unzip images.zip
cd ../../
```
After preparation is complete, the directory structure is:
```shell
data/MIRB
├── images
├── ...
├── visual_chain.json
└── visual_chain_concat.json
```
## 🏃 Evaluation Execution
> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
To run the evaluation, execute the following command on an 8-GPU setup:
```shell
torchrun --nproc_per_node=8 eval/mirb/evaluate_mirb.py --checkpoint ${CHECKPOINT} --dynamic
```
Alternatively, you can run the following simplified command:
```shell
GPUS=8 sh evaluate.sh ${CHECKPOINT} mirb --dynamic
```
### Arguments
The following arguments can be configured for the evaluation script:
| Argument | Type | Default | Description |
| ---------------- | ------ | -------- | ----------------------------------------------------------------------------------------------------------------- |
| `--checkpoint` | `str` | `''` | Path to the model checkpoint. |
| `--datasets` | `str` | `'MIRB'` | Comma-separated list of datasets to evaluate. |
| `--dynamic` | `flag` | `False` | Enables dynamic high resolution preprocessing. |
| `--max-num` | `int` | `6` | Maximum tile number for dynamic high resolution. |
| `--load-in-8bit` | `flag` | `False` | Loads the model weights in 8-bit precision. |
| `--auto` | `flag` | `False` | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |
internvl_chat/eval/mirb/evaluate_mirb.py 0 → 100644
View file @ 26e59280
import argparse
import itertools
import json
import os
import random
import re
import time
from functools import partial
import numpy as np
import torch
from internvl.model import load_model_and_tokenizer
from internvl.train.dataset import build_transform, dynamic_preprocess
from PIL import Image
from tqdm import tqdm
ds_collections = {
'MIRB': {
'root': 'data/MIRB',
'max_new_tokens': 512,
'min_new_tokens': 1,
'split': 'test'
},
}
word_to_num = {
'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4',
'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9', 'ten': '10',
'first': '1', 'second': '2', 'third': '3', 'fourth': '4', 'fifth': '5',
'sixth': '6', 'seventh': '7', 'eighth': '8', 'ninth': '9', 'tenth': '10'
}
# Define the groupings
groups = {
'Knowledge': ['food', 'sightseeing'],
'Reasoning': ['codeu', 'plot_code', 'analogy', '3d_scene'],
'Perception': ['image_jigsaw', 'count', 'attribute'],
'Multi-Hop': ['visual_chain', 'arxiv']
}
def eval_scores(results, dataset):
if dataset in ['count', 'codeu', 'food', 'image_jigsaw', 'arxiv', 'visual_chain', 'visual_chain_concat',
'plot_code', '3d_scene', '3d_scene_concat', 'count_concat', 'image_needles',
'image_needles_concat', 'plot_text', 'arxiv_text', 'codeu_text']:
score = exact_match(results, dataset)
elif dataset in ['analogy', 'attribute']:
score = exact_yes_no(results)
elif dataset in ['sightseeing']:
score = exact_in_match(results)
return score
def exact_yes_no(results):
acc = []
for result in results:
prediction = result['prediction'].strip()
prediction = prediction.strip('\n')
trunc_index = prediction.find('\n')
if trunc_index <= 0:
trunc_index = prediction.find('.')
if trunc_index > 0:
prediction = prediction[:trunc_index]
if result['answers'].lower() == 'yes' and 'yes' in str(prediction).lower():
acc.append(1)
elif result['answers'].lower() == 'no' and 'yes' not in str(prediction).lower():
acc.append(1)
else:
acc.append(0)
avg_acc = np.average(acc)
return avg_acc
def exact_in_match(results):
acc = []
for result in results:
if result['answers'].lower() in ['yes', 'no']:
prediction = result['prediction'].strip()
prediction = prediction.strip('\n')
trunc_index = prediction.find('\n')
if trunc_index <= 0:
trunc_index = prediction.find('.')
if trunc_index > 0:
prediction = prediction[:trunc_index]
if result['answers'].lower() == 'yes' and 'yes' in str(prediction).lower():
acc.append(1)
elif result['answers'].lower() == 'no' and 'yes' not in str(prediction).lower():
acc.append(1)
else:
acc.append(0)
continue
prediction = result['prediction'].strip()
prediction = prediction.strip('\n')
trunc_index = prediction.find('\n')
if trunc_index <= 0:
trunc_index = prediction.find('.')
if trunc_index > 0:
prediction = prediction[:trunc_index]
if str(result['answers']).lower() in str(prediction).lower():
acc.append(1)
else:
acc.append(0)
avg_acc = np.average(acc)
return avg_acc
def exact_match(results, dataset):
acc = []
for result in results:
prediction = result['prediction'].strip()
prediction = prediction.strip('\n')
trunc_index = prediction.find('\n')
if trunc_index <= 0:
trunc_index = prediction.find('.')
if trunc_index > 0:
prediction = prediction[:trunc_index]
if dataset in ['count', 'count_concat',
'visual_chain', 'visual_chain_concat',
'3d_scene', '3d_scene_concat',
'image_needles', 'image_needles_concat']:
# find the number
match = re.search(r'\d+', prediction)
if match:
prediction = match.group()
else:
if str(prediction.lower()) in word_to_num:
prediction = word_to_num[str(prediction.lower())]
else:
prediction = ''
if str(prediction).lower() == str(result['answers']).lower():
acc.append(1)
else:
acc.append(0)
avg_acc = np.average(acc)
return avg_acc
def collate_fn(batches, tokenizer):
try:
pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
except:
pixel_values = [None for _ in batches]
questions = [_['question'] for _ in batches]
answers = [_['answer'] for _ in batches]
num_patches_lists = [_['num_patches_list'] for _ in batches]
lines = [_['line'] for _ in batches]
return pixel_values, questions, answers, num_patches_lists, lines
def get_task_instruction(dataset):
if dataset in ['analogy', 'attribute', 'plot_code', 'plot_text', 'sightseeing',
'image_needles', 'image_needles_concat', 'visual_chain', 'visual_chain_concat']:
instr = 'Answer with a single word.'
elif dataset in ['codeu', 'food', 'image_jigsaw', 'codeu_text']:
instr = 'Answer with the option symbol.'
elif dataset in ['arxiv', 'arxiv_text']:
instr = 'Answer with the paper title.'
elif dataset in ['count', 'count_concat']:
instr = 'Answer with a single number.'
elif dataset in ['3d_scene', '3d_scene_concat']:
instr = 'The following images are different views of the same 3D scene. Answer with a single number.'
return instr
class MIRBDataset(torch.utils.data.Dataset):
def __init__(self, root, split, input_size=224, dynamic_image_size=False,
use_thumbnail=False, max_num=6):
json_files = [item for item in os.listdir(root) if item.endswith('.json')]
self.data = []
for json_file in json_files:
json_path = os.path.join(root, json_file)
# if '_concat' in json_path:
# continue
task = os.path.basename(json_path).replace('.json', '')
temp = json.loads(open(json_path).read())
for item in temp:
item['task'] = task
self.data.append(item)
self.root = root
self.input_size = input_size
self.dynamic_image_size = dynamic_image_size
self.use_thumbnail = use_thumbnail
self.max_num = max_num
self.transform = build_transform(is_train=False, input_size=input_size)
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
data = self.data[idx]
questions = data['questions']
answers = str(data['answers'])
task = data['task']
prompt = get_task_instruction(task)
question = questions + '\n' + prompt
input_image_path = data['images']
image_list = []
for image_path in input_image_path:
image_path = os.path.join(self.root, image_path)
image = Image.open(image_path).convert('RGB')
image_list.append(image)
num_patches_list = []
if self.dynamic_image_size:
images = []
for image in image_list:
tiles = dynamic_preprocess(image, image_size=self.input_size,
use_thumbnail=self.use_thumbnail,
max_num=max(1, self.max_num // len(image_list)))
images += tiles
num_patches_list.append(len(tiles))
else:
images = image_list
num_patches_list.append(1)
pixel_values = [self.transform(image) for image in images]
if len(pixel_values) > 0:
pixel_values = torch.stack(pixel_values)
else:
pixel_values = None
if len(image_list) == 1:
prefix = '<image>\n'
else:
prefix = ''.join([f'Image-{i + 1}: <image>\n' for i in range(len(image_list))])
question = prefix + question
return {
'question': question,
'pixel_values': pixel_values,
'answer': answers,
'num_patches_list': num_patches_list,
'line': data
}
class InferenceSampler(torch.utils.data.sampler.Sampler):
def __init__(self, size):
self._size = int(size)
assert size > 0
self._rank = torch.distributed.get_rank()
self._world_size = torch.distributed.get_world_size()
self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
@staticmethod
def _get_local_indices(total_size, world_size, rank):
shard_size = total_size // world_size
left = total_size % world_size
shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
begin = sum(shard_sizes[:rank])
end = min(sum(shard_sizes[:rank + 1]), total_size)
return range(begin, end)
def __iter__(self):
yield from self._local_indices
def __len__(self):
return len(self._local_indices)
def evaluate_chat_model():
random.seed(args.seed)
for ds_name in args.datasets:
dataset = MIRBDataset(
root=ds_collections[ds_name]['root'],
split=ds_collections[ds_name]['split'],
input_size=image_size,
dynamic_image_size=args.dynamic,
use_thumbnail=use_thumbnail,
max_num=args.max_num
)
dataloader = torch.utils.data.DataLoader(
dataset=dataset,
sampler=InferenceSampler(len(dataset)),
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
drop_last=False,
collate_fn=partial(collate_fn, tokenizer=tokenizer),
)
outputs = []
for _, (pixel_values, questions, answers, num_patches_lists, lines) in tqdm(enumerate(dataloader)):
try:
pixel_values = pixel_values.to(torch.bfloat16).cuda()
except:
pixel_values = None
generation_config = dict(
num_beams=args.num_beams,
max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
do_sample=True if args.temperature > 0 else False,
temperature=args.temperature,
)
try:
pred = model.chat(
tokenizer=tokenizer,
pixel_values=pixel_values,
question=questions[0],
generation_config=generation_config,
num_patches_list=num_patches_lists[0],
verbose=False
)
except:
pred = 'Error'
preds = [pred]
for question, pred, answer, line in zip(questions, preds, answers, lines):
task = line['task']
line = {
'question': question,
'prediction': pred,
'answers': answer,
'task': task
}
score = eval_scores([line], task)
line['score'] = score
outputs.append(line)
torch.distributed.barrier()
world_size = torch.distributed.get_world_size()
merged_outputs = [None for _ in range(world_size)]
torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
merged_outputs = [json.loads(_) for _ in merged_outputs]
merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
if torch.distributed.get_rank() == 0:
print(f'Evaluating {ds_name} ...')
time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
results_file = f'{ds_name}_{time_prefix}.jsonl'
output_path = os.path.join(args.out_dir, results_file)
writer = open(output_path, 'w')
scores = {}
for item in merged_outputs:
task = item['task']
score = item['score']
if task not in scores:
scores[task] = []
scores[task].append(score)
writer.write(json.dumps(item) + '\n')
writer.close()
print('Results saved to {}'.format(output_path))
averages = {}
for group_name, group_list in groups.items():
values = [np.mean(scores[task]) for task in group_list]
averages[group_name] = np.mean(values)
for category in averages:
print(f'{category} Acc: {averages[category]}')
print(f'Mean Acc: {np.mean(list(averages.values()))}')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--checkpoint', type=str, default='')
parser.add_argument('--datasets', type=str, default='MIRB')
parser.add_argument('--batch-size', type=int, default=1)
parser.add_argument('--num-workers', type=int, default=1)
parser.add_argument('--num-beams', type=int, default=1)
parser.add_argument('--temperature', type=float, default=0.0)
parser.add_argument('--out-dir', type=str, default='results')
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--dynamic', action='store_true')
parser.add_argument('--max-num', type=int, default=6)
parser.add_argument('--load-in-8bit', action='store_true')
parser.add_argument('--load-in-4bit', action='store_true')
parser.add_argument('--auto', action='store_true')
args = parser.parse_args()
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir, exist_ok=True)
args.datasets = args.datasets.split(',')
print('datasets:', args.datasets)
assert args.batch_size == 1, 'Only batch size 1 is supported'
torch.distributed.init_process_group(
backend='nccl',
world_size=int(os.getenv('WORLD_SIZE', '1')),
rank=int(os.getenv('RANK', '0')),
)
torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
model, tokenizer = load_model_and_tokenizer(args)
image_size = model.config.force_image_size or model.config.vision_config.image_size
use_thumbnail = model.config.use_thumbnail
total_params = sum(p.numel() for p in model.parameters()) / 1e9
if total_params > 20 or args.dynamic:
args.num_beams = 1
print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
else:
print(f'[test] total_params: {total_params}B')
print(f'[test] image_size: {image_size}')
print(f'[test] template: {model.config.template}')
print(f'[test] dynamic_image_size: {args.dynamic}')
print(f'[test] use_thumbnail: {use_thumbnail}')
print(f'[test] max_num: {args.max_num}')
evaluate_chat_model()
internvl_chat/eval/mmbench/README.md 0 → 100644
View file @ 26e59280
# README for Evaluation
## 🌟 Overview
This script provides an evaluation pipeline for `MMBench` and `CCBench`.
## 🗂️ Data Preparation
Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
### MMBench and CCBench
Follow the instructions below to prepare the data:
```shell
# Step 1: Create the data directory
mkdir -p data/mmbench && cd data/mmbench
# Step 2: Download csv files
wget http://opencompass.openxlab.space/utils/MMBench/CCBench_legacy.tsv
wget https://download.openmmlab.com/mmclassification/datasets/mmbench/mmbench_dev_20230712.tsv
wget https://download.openmmlab.com/mmclassification/datasets/mmbench/mmbench_dev_cn_20231003.tsv
wget https://download.openmmlab.com/mmclassification/datasets/mmbench/mmbench_dev_en_20231003.tsv
wget https://download.openmmlab.com/mmclassification/datasets/mmbench/mmbench_test_cn_20231003.tsv
wget https://download.openmmlab.com/mmclassification/datasets/mmbench/mmbench_test_en_20231003.tsv
cd ../..
```
After preparation is complete, the directory structure is:
```shell
data/mmbench
├── CCBench_legacy.tsv
├── mmbench_dev_20230712.tsv
├── mmbench_dev_cn_20231003.tsv
├── mmbench_dev_en_20231003.tsv
├── mmbench_test_cn_20231003.tsv
└── mmbench_test_en_20231003.tsv
```
## 🏃 Evaluation Execution
> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
To run the evaluation, execute the following command on an 8-GPU setup:
```shell
# Test the MMBench-Dev-EN
torchrun --nproc_per_node=8 eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --dynamic --datasets mmbench_dev_20230712
# Test the MMBench-Test-EN
torchrun --nproc_per_node=8 eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --dynamic --datasets mmbench_test_en_20231003
# Test the MMBench-Dev-CN
torchrun --nproc_per_node=8 eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --dynamic --datasets mmbench_dev_cn_20231003
# Test the MMBench-Test-CN
torchrun --nproc_per_node=8 eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --dynamic --datasets mmbench_test_cn_20231003
# Test the CCBench-Dev
torchrun --nproc_per_node=8 eval/mmbench/evaluate_mmbench.py --checkpoint ${CHECKPOINT} --dynamic --datasets ccbench_dev_cn
```
Alternatively, you can run the following simplified command:
```shell
# Test the MMBench-Dev-EN
GPUS=8 sh evaluate.sh ${CHECKPOINT} mmbench-dev-en --dynamic
# Test the MMBench-Test-EN
GPUS=8 sh evaluate.sh ${CHECKPOINT} mmbench-test-en --dynamic
# Test the MMBench-Dev-CN
GPUS=8 sh evaluate.sh ${CHECKPOINT} mmbench-dev-cn --dynamic
# Test the MMBench-Test-CN
GPUS=8 sh evaluate.sh ${CHECKPOINT} mmbench-test-cn --dynamic
# Test the CCBench-Dev
GPUS=8 sh evaluate.sh ${CHECKPOINT} ccbench-dev --dynamic
```
After the test is completed, a file with a name similar to `results/mmbench_dev_20230712_241224214015.xlsx` will be generated. Please upload these files to the [official server](https://mmbench.opencompass.org.cn/mmbench-submission) to obtain the evaluation scores.
### Arguments
The following arguments can be configured for the evaluation script:
| Argument | Type | Default | Description |
| ---------------- | ------ | ------------------------ | ----------------------------------------------------------------------------------------------------------------- |
| `--checkpoint` | `str` | `''` | Path to the model checkpoint. |
| `--datasets` | `str` | `'mmbench_dev_20230712'` | Comma-separated list of datasets to evaluate. |
| `--dynamic` | `flag` | `False` | Enables dynamic high resolution preprocessing. |
| `--max-num` | `int` | `6` | Maximum tile number for dynamic high resolution. |
| `--load-in-8bit` | `flag` | `False` | Loads the model weights in 8-bit precision. |
| `--auto` | `flag` | `False` | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |
internvl_chat/eval/mmbench/evaluate_mmbench.py 0 → 100644
View file @ 26e59280
import argparse
import base64
import itertools
import json
import os
import random
import time
from functools import partial
from io import BytesIO
import pandas as pd
import torch
from internvl.model import load_model_and_tokenizer
from internvl.train.dataset import build_transform, dynamic_preprocess
from PIL import Image
from tqdm import tqdm
ds_collections = {
'mmbench_dev_20230712': {
'root': 'data/mmbench/mmbench_dev_20230712.tsv',
'max_new_tokens': 100,
'min_new_tokens': 1,
'type': 'dev',
'language': 'en'
},
'mmbench_dev_cn_20231003': {
'root': 'data/mmbench/mmbench_dev_cn_20231003.tsv',
'max_new_tokens': 100,
'min_new_tokens': 1,
'type': 'dev',
'language': 'cn'
},
'mmbench_dev_en_20231003': {
'root': 'data/mmbench/mmbench_dev_en_20231003.tsv',
'max_new_tokens': 100,
'min_new_tokens': 1,
'type': 'dev',
'language': 'en'
},
'mmbench_test_cn_20231003': {
'root': 'data/mmbench/mmbench_test_cn_20231003.tsv',
'max_new_tokens': 100,
'min_new_tokens': 1,
'type': 'test',
'language': 'cn'
},
'mmbench_test_en_20231003': {
'root': 'data/mmbench/mmbench_test_en_20231003.tsv',
'max_new_tokens': 100,
'min_new_tokens': 1,
'type': 'test',
'language': 'en'
},
'ccbench_dev_cn': {
'root': 'data/mmbench/CCBench_legacy.tsv',
'max_new_tokens': 100,
'min_new_tokens': 1,
'type': 'dev',
'language': 'cn'
}
}
def collate_fn(batches, tokenizer):
pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
questions = [_['question'] for _ in batches]
answers = [_['answer'] for _ in batches]
indexes = [_['index'] for _ in batches]
options = [_['option'] for _ in batches]
return pixel_values, questions, answers, indexes, options
class MMBenchDataset(torch.utils.data.Dataset):
def __init__(self, root, prompt, language, input_size=224, dynamic_image_size=False,
use_thumbnail=False, max_num=6):
self.df = pd.read_csv(root, sep='\t')
self.prompt = prompt
self.language = language
self.input_size = input_size
self.dynamic_image_size = dynamic_image_size
self.use_thumbnail = use_thumbnail
self.max_num = max_num
self.transform = build_transform(is_train=False, input_size=input_size)
def __len__(self):
return len(self.df)
def __getitem__(self, idx):
index = self.df.iloc[idx]['index']
image = self.df.iloc[idx]['image']
question = self.df.iloc[idx]['question']
answer = self.df.iloc[idx]['answer'] if 'answer' in self.df.iloc[0].keys() else None
# catetory = self.df.iloc[idx]['category']
# l2_catetory = self.df.iloc[idx]['l2-category']
image = Image.open(BytesIO(base64.b64decode(image))).convert('RGB')
if self.dynamic_image_size:
images = dynamic_preprocess(image, image_size=self.input_size,
use_thumbnail=self.use_thumbnail,
max_num=self.max_num)
else:
images = [image]
pixel_values = [self.transform(image) for image in images]
pixel_values = torch.stack(pixel_values)
option_candidate = ['A', 'B', 'C', 'D', 'E']
options = {
cand: self.load_from_df(idx, cand)
for cand in option_candidate
if self.load_from_df(idx, cand) is not None
}
hint = self.load_from_df(idx, 'hint')
if hint is not None:
question = hint + '\n' + question
for key, item in options.items():
question += f'\n{key}. {item}'
if self.language == 'cn':
question = question + '\n' + self.prompt['cn']
else:
question = question + '\n' + self.prompt['en']
return {
'question': question,
'pixel_values': pixel_values,
'answer': answer,
'index': index,
'option': options
}
def load_from_df(self, idx, key):
if key in self.df.iloc[idx] and not pd.isna(self.df.iloc[idx][key]):
return self.df.iloc[idx][key]
else:
return None
class InferenceSampler(torch.utils.data.sampler.Sampler):
def __init__(self, size):
self._size = int(size)
assert size > 0
self._rank = torch.distributed.get_rank()
self._world_size = torch.distributed.get_world_size()
self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
@staticmethod
def _get_local_indices(total_size, world_size, rank):
shard_size = total_size // world_size
left = total_size % world_size
shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
begin = sum(shard_sizes[:rank])
end = min(sum(shard_sizes[:rank + 1]), total_size)
return range(begin, end)
def __iter__(self):
yield from self._local_indices
def __len__(self):
return len(self._local_indices)
def post_process(pred, option):
pred = pred.strip()
option_candidate = list(option.keys())
if len(pred) == 1:
return pred
elif len(pred) != 1 and pred[0] in option_candidate:
return pred[0]
elif len(pred) != 1 and pred[0] not in option_candidate:
for k, v in option.items():
if v in pred:
return k
return pred
def evaluate_chat_model():
random.seed(args.seed)
for ds_name in args.datasets:
dataset = MMBenchDataset(
root=ds_collections[ds_name]['root'],
prompt=prompt,
language=ds_collections[ds_name]['language'],
input_size=image_size,
dynamic_image_size=args.dynamic,
use_thumbnail=use_thumbnail,
max_num=args.max_num
)
dataloader = torch.utils.data.DataLoader(
dataset=dataset,
sampler=InferenceSampler(len(dataset)),
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
drop_last=False,
collate_fn=partial(collate_fn, tokenizer=tokenizer),
)
outputs = []
for _, (pixel_values, questions, answers, indexes, options) in tqdm(enumerate(dataloader)):
pixel_values = pixel_values.to(torch.bfloat16).cuda()
generation_config = dict(
num_beams=args.num_beams,
max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
do_sample=True if args.temperature > 0 else False,
temperature=args.temperature,
)
pred = model.chat(
tokenizer=tokenizer,
pixel_values=pixel_values,
question=questions[0],
generation_config=generation_config,
verbose=True
)
preds = [post_process(pred, options[0])]
for question, pred, answer, index in zip(questions, preds, answers, indexes):
outputs.append({
'question': question,
'answer': pred,
'gt_answers': answer,
'index': int(index)
})
torch.distributed.barrier()
world_size = torch.distributed.get_world_size()
merged_outputs = [None for _ in range(world_size)]
torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
merged_outputs = [json.loads(_) for _ in merged_outputs]
merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
if torch.distributed.get_rank() == 0:
print(f'Evaluating {ds_name} ...')
time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
results_file = f'{ds_name}_{time_prefix}.xlsx'
output_path = os.path.join(args.out_dir, results_file)
df = pd.read_table(ds_collections[ds_name]['root'])
cur_df = df.copy()
if 'mmbench' in ds_name:
cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
cur_df.insert(6, 'prediction', None)
else:
cur_df = cur_df.drop(columns=['category', 'image'])
cur_df.insert(8, 'prediction', None)
for item in merged_outputs:
cur_df.loc[df['index'] == item['index'], 'prediction'] = item['answer']
cur_df.to_excel(output_path, index=False, engine='openpyxl')
print('Results saved to {}'.format(output_path))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--checkpoint', type=str, default='')
parser.add_argument('--datasets', type=str, default='mmbench_dev_20230712')
parser.add_argument('--batch-size', type=int, default=1)
parser.add_argument('--num-workers', type=int, default=1)
parser.add_argument('--num-beams', type=int, default=1)
parser.add_argument('--temperature', type=float, default=0.0)
parser.add_argument('--out-dir', type=str, default='results')
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--dynamic', action='store_true')
parser.add_argument('--max-num', type=int, default=6)
parser.add_argument('--load-in-8bit', action='store_true')
parser.add_argument('--load-in-4bit', action='store_true')
parser.add_argument('--auto', action='store_true')
args = parser.parse_args()
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir, exist_ok=True)
args.datasets = args.datasets.split(',')
print('datasets:', args.datasets)
assert args.batch_size == 1, 'Only batch size 1 is supported'
torch.distributed.init_process_group(
backend='nccl',
world_size=int(os.getenv('WORLD_SIZE', '1')),
rank=int(os.getenv('RANK', '0')),
)
torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
model, tokenizer = load_model_and_tokenizer(args)
image_size = model.config.force_image_size or model.config.vision_config.image_size
use_thumbnail = model.config.use_thumbnail
total_params = sum(p.numel() for p in model.parameters()) / 1e9
if total_params > 20 or args.dynamic:
args.num_beams = 1
print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
else:
print(f'[test] total_params: {total_params}B')
print(f'[test] image_size: {image_size}')
print(f'[test] template: {model.config.template}')
print(f'[test] dynamic_image_size: {args.dynamic}')
print(f'[test] use_thumbnail: {use_thumbnail}')
print(f'[test] max_num: {args.max_num}')
prompt = {
'en': "Answer with the option's letter from the given choices directly.",
'cn': '请直接回答选项字母。'
}
evaluate_chat_model()
internvl_chat/eval/mme/README.md 0 → 100644
View file @ 26e59280
# README for Evaluation
## 🌟 Overview
This script provides an evaluation pipeline for `MME`.
## 🗂️ Data Preparation
Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
### MME
Follow the instructions below to prepare the data:
```shell
# Step 1: Create the data directory
mkdir -p data/mme && cd data/mme
# Step 2: Download MME_Benchmark_release_version
wget https://huggingface.co/OpenGVLab/InternVL/resolve/main/MME_Benchmark_release_version.zip
unzip MME_Benchmark_release_version.zip
cd ../..
```
After preparation is complete, the directory structure is:
```shell
data/mme
└── MME_Benchmark_release_version
```
## 🏃 Evaluation Execution
> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
To run the evaluation, execute the following command on an 1-GPU setup:
```shell
cd eval/mme/
DIRNAME=`basename ${CHECKPOINT}`
python eval.py --checkpoint ${CHECKPOINT} --dynamic
python calculation.py --results_dir ${DIRNAME}
cd ../../
```
Alternatively, you can run the following simplified command:
```shell
GPUS=1 sh evaluate.sh ${CHECKPOINT} mme --dynamic
```
### Arguments
The following arguments can be configured for the evaluation script:
| Argument | Type | Default | Description |
| ---------------- | ------ | ------- | ----------------------------------------------------------------------------------------------------------------- |
| `--checkpoint` | `str` | `''` | Path to the model checkpoint. |
| `--dynamic` | `flag` | `False` | Enables dynamic high resolution preprocessing. |
| `--max-num` | `int` | `6` | Maximum tile number for dynamic high resolution. |
| `--load-in-8bit` | `flag` | `False` | Loads the model weights in 8-bit precision. |
| `--auto` | `flag` | `False` | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |
internvl_chat/eval/mme/Your_Results/OCR.txt 0 → 100644
View file @ 26e59280
0001.jpg Is the word in the logo "angie's"? Please answer yes or no. Yes
0001.jpg Is the word in the logo "angle's"? Please answer yes or no. No
0002.jpg Is the word in the logo "c'est cheese"? Please answer yes or no. Yes
0002.jpg Is the word in the logo "crest cheese"? Please answer yes or no. No
0003.jpg Is the word in the logo "beavertails pastry"? Please answer yes or no. Yes
0003.jpg Is the word in the logo "beavertalls pastry"? Please answer yes or no. No
0004.jpg Is the word in the logo "old market sundries"? Please answer yes or no. Yes
0004.jpg Is the word in the logo "old market hundreds"? Please answer yes or no. No
0005.jpg Is the word in the logo "kress"? Please answer yes or no. Yes
0005.jpg Is the word in the logo "dress"? Please answer yes or no. No
0006.jpg Is the word in the logo "the beatles story liver pool"? Please answer yes or no. Yes
0006.jpg Is the word in the logo "the beats story liver pool"? Please answer yes or no. No
0007.jpg Is the phone number in the picture "0131 555 6363"? Please answer yes or no. Yes
0007.jpg Is the phone number in the picture "0137 556 6363"? Please answer yes or no. No
0008.jpg Is the word in the logo "phil's market"? Please answer yes or no. Yes
0008.jpg Is the word in the logo "phll's market"? Please answer yes or no. No
0009.jpg Is the word in the logo "fenders diner"? Please answer yes or no. Yes
0009.jpg Is the word in the logo "finders diner"? Please answer yes or no. No
0010.jpg Is the word in the logo "high time coffee shop"? Please answer yes or no. Yes
0010.jpg Is the word in the logo "high tite cofeee shop"? Please answer yes or no. No
0011.jpg Is the word in the logo "ihop restaurant"? Please answer yes or no. Yes
0011.jpg Is the word in the logo "lhop restaurant"? Please answer yes or no. No
0012.jpg Is the word in the logo "casa grecque restaurants"? Please answer yes or no. Yes
0012.jpg Is the word in the logo "case grecque restaurants"? Please answer yes or no. No
0013.jpg Is the word in the picture "seabreeze motel"? Please answer yes or no. Yes
0013.jpg Is the word in the picture "seebreeze model"? Please answer yes or no. No
0014.jpg Is the word in the logo "penarth pier built 1894"? Please answer yes or no. Yes
0014.jpg Is the word in the logo "penarth pies buid 1894"? Please answer yes or no. No
0015.jpg Is the text in the picture "hollywood"? Please answer yes or no. Yes
0015.jpg Is the text in the picture "holly word"? Please answer yes or no. No
0016.jpg Is the word in the logo "shop rite"? Please answer yes or no. Yes
0016.jpg Is the word in the logo "stop rite"? Please answer yes or no. No
0017.jpg Is the word in the logo "hardco industrial construction"? Please answer yes or no. Yes
0017.jpg Is the word in the logo "hardto industal construction"? Please answer yes or no. No
0018.jpg Is the word in the logo "oldsmobile service"? Please answer yes or no. Yes
0018.jpg Is the word in the logo "old mobile service"? Please answer yes or no. No
0019.jpg Is the word in the logo "exchange hotel"? Please answer yes or no. Yes
0019.jpg Is the word in the logo "excharge hotel"? Please answer yes or no. No
0020.jpg Is the word in the logo "cold drinks"? Please answer yes or no. Yes
0020.jpg Is the word in the logo "cold rinks"? Please answer yes or no. No
internvl_chat/eval/mme/Your_Results/artwork.txt 0 → 100644
View file @ 26e59280
10002.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
10002.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No
10049.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
10049.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. No
10256.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
10256.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. No
10358.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
10358.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No
10543.jpg Is this artwork displayed in fogg art museum, harvard university, cambridge? Please answer yes or no. Yes
10543.jpg Is this artwork displayed in museo civico, pistoia? Please answer yes or no. No
10581.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes
10581.jpg Does this artwork belong to the type of genre? Please answer yes or no. No
1060.jpg Is this artwork created by antoniazzo romano? Please answer yes or no. Yes
1060.jpg Is this artwork created by gentile da fabriano? Please answer yes or no. No
10881.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
10881.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No
10970.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
10970.jpg Does this artwork belong to the type of study? Please answer yes or no. No
11276.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes
11276.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No
11331.jpg Is this artwork created by donatello? Please answer yes or no. Yes
11331.jpg Is this artwork created by zichy, mihály? Please answer yes or no. No
11488.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes
11488.jpg Does this artwork belong to the type of historical? Please answer yes or no. No
11724.jpg Is this artwork created by duccio di buoninsegna? Please answer yes or no. Yes
11724.jpg Is this artwork created by giani, felice? Please answer yes or no. No
11726.jpg Is this artwork titled temptation on the mountain (detail)? Please answer yes or no. Yes
11726.jpg Is this artwork titled in the forest of fontainebleau? Please answer yes or no. No
12133.jpg Is this artwork titled hand study with bible? Please answer yes or no. Yes
12133.jpg Is this artwork titled self-portrait aged 78? Please answer yes or no. No
12439.jpg Is this artwork created by dürer, albrecht? Please answer yes or no. Yes
12439.jpg Is this artwork created by koekkoek, barend cornelis? Please answer yes or no. No
12561.jpg Is this artwork created by eberlein, gustav heinrich? Please answer yes or no. Yes
12561.jpg Is this artwork created by gillemans, jan pauwel the younger? Please answer yes or no. No
12652.jpg Is this artwork displayed in stedelijk museum de lakenhal, leiden? Please answer yes or no. Yes
12652.jpg Is this artwork displayed in palazzo ducale, mantua? Please answer yes or no. No
12736.jpg Is this artwork displayed in cannon hall museum, barnsley? Please answer yes or no. Yes
12736.jpg Is this artwork displayed in protestant parish church, gelnhausen? Please answer yes or no. No
12902.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
12902.jpg Is this artwork displayed in musée national gustave-moreau, paris? Please answer yes or no. No
12908.jpg Is this artwork titled ruth and boaz? Please answer yes or no. Yes
12908.jpg Is this artwork titled view of dresden from the right bank of the elbe with the augustus bridge? Please answer yes or no. No
13091.jpg Is this artwork titled italianate landscape with figures by classical ruins? Please answer yes or no. Yes
13091.jpg Is this artwork titled two boys singing? Please answer yes or no. No
13174.jpg Is this artwork titled nobility? Please answer yes or no. Yes
13174.jpg Is this artwork titled aretino in the studio of tintoretto? Please answer yes or no. No
13239.jpg Is this artwork titled doge ziani receiving the benediction of pope alexander iii? Please answer yes or no. Yes
13239.jpg Is this artwork titled the adoration of the shepherds? Please answer yes or no. No
13288.jpg Does this artwork exist in the form of architecture? Please answer yes or no. Yes
13288.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
13696.jpg Is this artwork displayed in pinacoteca nazionale, siena? Please answer yes or no. Yes
13696.jpg Is this artwork displayed in british embassy, paris? Please answer yes or no. No
13760.jpg Is this artwork titled noli me tangere? Please answer yes or no. Yes
13760.jpg Is this artwork titled profile study of a bearded man? Please answer yes or no. No
13821.jpg Is this artwork created by frangipane, niccolò? Please answer yes or no. Yes
13821.jpg Is this artwork created by drevet, pierre? Please answer yes or no. No
13901.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
13901.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
14283.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
14283.jpg Does this artwork exist in the form of mosaic? Please answer yes or no. No
14499.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
14499.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No
14777.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
14777.jpg Does this artwork belong to the type of historical? Please answer yes or no. No
15028.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes
15028.jpg Does this artwork belong to the type of study? Please answer yes or no. No
15232.jpg Is this artwork created by giordano, luca? Please answer yes or no. Yes
15232.jpg Is this artwork created by heyerdahl, hans olaf? Please answer yes or no. No
15246.jpg Is this artwork displayed in palazzo medici riccardi, florence? Please answer yes or no. Yes
15246.jpg Is this artwork displayed in abbey church of sainte-foy, conques (aveyron)? Please answer yes or no. No
15311.jpg Is this artwork created by giorgione? Please answer yes or no. Yes
15311.jpg Is this artwork created by marilhat, prosper? Please answer yes or no. No
15989.jpg Is this artwork displayed in pinacoteca, vatican? Please answer yes or no. Yes
15989.jpg Is this artwork displayed in cathedral museum, zamora? Please answer yes or no. No
16006.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
16006.jpg Is this artwork displayed in cathedral of san geminiano, modena? Please answer yes or no. No
16249.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes
16249.jpg Does this artwork belong to the type of religious? Please answer yes or no. No
16538.jpg Is this artwork created by gogh, vincent van? Please answer yes or no. Yes
16538.jpg Is this artwork created by altdorfer, albrecht? Please answer yes or no. No
16835.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
16835.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No
16911.jpg Is this artwork created by gossart, jan? Please answer yes or no. Yes
16911.jpg Is this artwork created by stanzione, massimo? Please answer yes or no. No
17311.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
17311.jpg Does this artwork belong to the type of interior? Please answer yes or no. No
17317.jpg Is this artwork created by gozzoli, benozzo? Please answer yes or no. Yes
17317.jpg Is this artwork created by coriolano, cristoforo? Please answer yes or no. No
17535.jpg Is this artwork created by grebber, pieter de? Please answer yes or no. Yes
17535.jpg Is this artwork created by massys, quentin? Please answer yes or no. No
17823.jpg Is this artwork created by greuze, jean-baptiste? Please answer yes or no. Yes
17823.jpg Is this artwork created by landseer, sir edwin henry? Please answer yes or no. No
17838.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
17838.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No
17998.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
17998.jpg Does this artwork belong to the type of genre? Please answer yes or no. No
18566.jpg Is this artwork created by hamen, juan van der? Please answer yes or no. Yes
18566.jpg Is this artwork created by starnina, gherardo di jacopo? Please answer yes or no. No
18604.jpg Is this artwork created by hardouin-mansart, jules? Please answer yes or no. Yes
18604.jpg Is this artwork created by kerseboom, friedrich? Please answer yes or no. No
18722.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
18722.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. No
1873.jpg Does this artwork exist in the form of architecture? Please answer yes or no. Yes
1873.jpg Does this artwork exist in the form of painting? Please answer yes or no. No
18902.jpg Is this artwork created by herrera, francisco de, the elder? Please answer yes or no. Yes
18902.jpg Is this artwork created by ingres, jean-auguste-dominique? Please answer yes or no. No
18926.jpg Is this artwork created by herring, john frederick the younger? Please answer yes or no. Yes
18926.jpg Is this artwork created by cozens, john robert? Please answer yes or no. No
19087.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
19087.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
19154.jpg Is this artwork titled portrait of the merchant georg gisze (detail)? Please answer yes or no. Yes
19154.jpg Is this artwork titled pair of table candlesticks? Please answer yes or no. No
19417.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
19417.jpg Does this artwork exist in the form of mosaic? Please answer yes or no. No
19452.jpg Is this artwork titled the artist and his model? Please answer yes or no. Yes
19452.jpg Is this artwork titled the lovesick maiden (detail)? Please answer yes or no. No
19839.jpg Is this artwork created by janneck, franz christoph? Please answer yes or no. Yes
19839.jpg Is this artwork created by goupil, jules-adolphe? Please answer yes or no. No
19863.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
19863.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No
19993.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
19993.jpg Is this artwork displayed in cathedral of st paul, liège? Please answer yes or no. No
20176.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
20176.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No
20437.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
20437.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No
20442.jpg Is this artwork created by kucharski, aleksander? Please answer yes or no. Yes
20442.jpg Is this artwork created by pourbus, frans the elder? Please answer yes or no. No
20455.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
20455.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
20483.jpg Is this artwork titled allegory of the regency? Please answer yes or no. Yes
20483.jpg Is this artwork titled breton woman bathing? Please answer yes or no. No
20490.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
20490.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No
20551.jpg Is this artwork created by lagrenée, jean-jacques? Please answer yes or no. Yes
20551.jpg Is this artwork created by scultori, diana? Please answer yes or no. No
20651.jpg Is this artwork titled a highland landscape? Please answer yes or no. Yes
20651.jpg Is this artwork titled a dog and a cat fighting in a kitchen interior? Please answer yes or no. No
20724.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes
20724.jpg Does this artwork belong to the type of landscape? Please answer yes or no. No
21048.jpg Is this artwork created by lemoyne, jean-baptiste ii? Please answer yes or no. Yes
21048.jpg Is this artwork created by kneller, sir godfrey? Please answer yes or no. No
21097.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
21097.jpg Does this artwork belong to the type of genre? Please answer yes or no. No
21244.jpg Does this artwork belong to the type of study? Please answer yes or no. Yes
21244.jpg Does this artwork belong to the type of portrait? Please answer yes or no. No
21469.jpg Does this artwork belong to the type of genre? Please answer yes or no. Yes
21469.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No
21580.jpg Is this artwork created by linard, jacques? Please answer yes or no. Yes
21580.jpg Is this artwork created by bonino da campione? Please answer yes or no. No
21712.jpg Is this artwork titled st john the evangelist resuscitating drusiana? Please answer yes or no. Yes
21712.jpg Is this artwork titled la finette? Please answer yes or no. No
22329.jpg Is this artwork titled marriage of the virgin? Please answer yes or no. Yes
22329.jpg Is this artwork titled landscape with river and figures (detail)? Please answer yes or no. No
22366.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
22366.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No
22667.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
22667.jpg Is this artwork displayed in san francesco d'assisi, pavia? Please answer yes or no. No
22760.jpg Is this artwork titled madonna and child (detail)? Please answer yes or no. Yes
22760.jpg Is this artwork titled view of the south and east walls? Please answer yes or no. No
22842.jpg Is this artwork titled ukrainian peasant girl? Please answer yes or no. Yes
22842.jpg Is this artwork titled virtue crowning merit? Please answer yes or no. No
23229.jpg Is this artwork displayed in national gallery, london? Please answer yes or no. Yes
23229.jpg Is this artwork displayed in notre-dame-la-riche, tours? Please answer yes or no. No
23427.jpg Is this artwork displayed in the hermitage, st. petersburg? Please answer yes or no. Yes
23427.jpg Is this artwork displayed in national gallery of victoria, melbourne? Please answer yes or no. No
23465.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
23465.jpg Is this artwork displayed in cistertian church, zirc? Please answer yes or no. No
23824.jpg Is this artwork titled christ walking on the water? Please answer yes or no. Yes
23824.jpg Is this artwork titled mademoiselle romaine lacaux? Please answer yes or no. No
24122.jpg Is this artwork displayed in museo correr, venice? Please answer yes or no. Yes
24122.jpg Is this artwork displayed in church of brou, bourg-en-bresse? Please answer yes or no. No
24260.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
24260.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No
24291.jpg Is this artwork titled virgin and child with sts catherine, cecilia, barbara, and ursula? Please answer yes or no. Yes
24291.jpg Is this artwork titled sorrow? Please answer yes or no. No
24723.jpg Is this artwork titled tomb of henry the lion and his wife matilda? Please answer yes or no. Yes
24723.jpg Is this artwork titled god the father? Please answer yes or no. No
2490.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes
2490.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No
2507.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
2507.jpg Is this artwork displayed in st. vitus's cathedral, prague? Please answer yes or no. No
25312.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
25312.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
25476.jpg Is this artwork created by michelangelo buonarroti? Please answer yes or no. Yes
25476.jpg Is this artwork created by beuckelaer, joachim? Please answer yes or no. No
25492.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes
25492.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No
25513.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
25513.jpg Does this artwork belong to the type of landscape? Please answer yes or no. No
26521.jpg Does this artwork exist in the form of illumination? Please answer yes or no. Yes
26521.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No
26973.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
26973.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No
27021.jpg Is this artwork created by miniaturist, german? Please answer yes or no. Yes
27021.jpg Is this artwork created by trinquesse, louis-rolland? Please answer yes or no. No
27662.jpg Does this artwork belong to the type of still-life? Please answer yes or no. Yes
27662.jpg Does this artwork belong to the type of mythological? Please answer yes or no. No
27936.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes
27936.jpg Does this artwork belong to the type of interior? Please answer yes or no. No
28039.jpg Is this artwork displayed in cappella palatina, palermo? Please answer yes or no. Yes
28039.jpg Is this artwork displayed in musée des beaux-arts, chambéry? Please answer yes or no. No
28345.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
28345.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No
28400.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
28400.jpg Does this artwork belong to the type of portrait? Please answer yes or no. No
28698.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
28698.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No
28758.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
28758.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No
28974.jpg Is this artwork titled prayer before the meal? Please answer yes or no. Yes
28974.jpg Is this artwork titled rest in the mountains? Please answer yes or no. No
29266.jpg Is this artwork created by palma vecchio? Please answer yes or no. Yes
29266.jpg Is this artwork created by maris, jacobus hendricus? Please answer yes or no. No
30443.jpg Is this artwork titled the crucifixion with sts jerome and christopher? Please answer yes or no. Yes
30443.jpg Is this artwork titled tomb of michelangelo (detail)? Please answer yes or no. No
3085.jpg Is this artwork created by bartsius, willem? Please answer yes or no. Yes
3085.jpg Is this artwork created by oehme, ernst ferdinand? Please answer yes or no. No
30875.jpg Is this artwork created by pomarancio? Please answer yes or no. Yes
30875.jpg Is this artwork created by steen, jan? Please answer yes or no. No
3114.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
3114.jpg Does this artwork belong to the type of study? Please answer yes or no. No
31808.jpg Is this artwork created by raffaello sanzio? Please answer yes or no. Yes
31808.jpg Is this artwork created by simon von taisten? Please answer yes or no. No
32147.jpg Is this artwork titled lucretia? Please answer yes or no. Yes
32147.jpg Is this artwork titled rinaldo abandoning armida (detail)? Please answer yes or no. No
3241.jpg Is this artwork titled holy family? Please answer yes or no. Yes
3241.jpg Is this artwork titled friedrich iii, the wise, and johann i, the constant, electors of saxony? Please answer yes or no. No
33017.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
33017.jpg Does this artwork exist in the form of glassware? Please answer yes or no. No
33069.jpg Does this artwork belong to the type of historical? Please answer yes or no. Yes
33069.jpg Does this artwork belong to the type of interior? Please answer yes or no. No
33173.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
33173.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No
33753.jpg Is this artwork titled vanitas? Please answer yes or no. Yes
33753.jpg Is this artwork titled legend of st francis: 18. apparition at arles (detail)? Please answer yes or no. No
33854.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
33854.jpg Does this artwork belong to the type of study? Please answer yes or no. No
339.jpg Is this artwork displayed in staatliche museen, berlin? Please answer yes or no. Yes
339.jpg Is this artwork displayed in national museum of religious carvings, valladolid? Please answer yes or no. No
33933.jpg Is this artwork titled madonna and child? Please answer yes or no. Yes
33933.jpg Is this artwork titled the bacino di san marco? Please answer yes or no. No
3404.jpg Is this artwork displayed in szépmûvészeti múzeum, budapest? Please answer yes or no. Yes
3404.jpg Is this artwork displayed in s. eustorgio, milan? Please answer yes or no. No
34109.jpg Is this artwork displayed in national gallery of art, washington? Please answer yes or no. Yes
34109.jpg Is this artwork displayed in abbey church of sainte-foy, conques? Please answer yes or no. No
34363.jpg Is this artwork displayed in museo del prado, madrid? Please answer yes or no. Yes
34363.jpg Is this artwork displayed in state tretyakov gallery, moscow? Please answer yes or no. No
34539.jpg Is this artwork titled the victory of eucharistic truth over heresy? Please answer yes or no. Yes
34539.jpg Is this artwork titled a sunday afternoon on the ile de la grande jatte? Please answer yes or no. No
34627.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes
34627.jpg Does this artwork belong to the type of genre? Please answer yes or no. No
34638.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
34638.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No
34669.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes
34669.jpg Does this artwork belong to the type of historical? Please answer yes or no. No
35345.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
35345.jpg Does this artwork belong to the type of landscape? Please answer yes or no. No
35439.jpg Is this artwork titled madonna and child with a host of musical angels? Please answer yes or no. Yes
35439.jpg Is this artwork titled garden in fontenay? Please answer yes or no. No
35460.jpg Is this artwork created by schinkel, karl friedrich? Please answer yes or no. Yes
35460.jpg Is this artwork created by giolfino, bartolomeo? Please answer yes or no. No
35486.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
35486.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No
35513.jpg Is this artwork created by schongauer, martin? Please answer yes or no. Yes
35513.jpg Is this artwork created by cassioli, amos? Please answer yes or no. No
3552.jpg Is this artwork titled madonna degli alberetti? Please answer yes or no. Yes
3552.jpg Is this artwork titled peter gillis? Please answer yes or no. No
35658.jpg Is this artwork created by sebastiano del piombo? Please answer yes or no. Yes
35658.jpg Is this artwork created by jacobsz., dirck? Please answer yes or no. No
35736.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
35736.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No
35861.jpg Does this artwork belong to the type of interior? Please answer yes or no. Yes
35861.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No
36805.jpg Is this artwork titled weir? Please answer yes or no. Yes
36805.jpg Is this artwork titled view of the window wall? Please answer yes or no. No
36966.jpg Does this artwork belong to the type of portrait? Please answer yes or no. Yes
36966.jpg Does this artwork belong to the type of religious? Please answer yes or no. No
37010.jpg Is this artwork titled madonna and child with the young st john? Please answer yes or no. Yes
37010.jpg Is this artwork titled sketch for attila? Please answer yes or no. No
37077.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
37077.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No
37439.jpg Is this artwork titled the message? Please answer yes or no. Yes
37439.jpg Is this artwork titled the descent from the cross? Please answer yes or no. No
37819.jpg Is this artwork created by tiepolo, giovanni battista? Please answer yes or no. Yes
37819.jpg Is this artwork created by kerricx, willem ignatius? Please answer yes or no. No
37866.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes
37866.jpg Does this artwork belong to the type of still-life? Please answer yes or no. No
381.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
381.jpg Does this artwork exist in the form of architecture? Please answer yes or no. No
38178.jpg Is this artwork created by tintoretto? Please answer yes or no. Yes
38178.jpg Is this artwork created by morel, jean-baptiste? Please answer yes or no. No
38536.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
38536.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No
38546.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
38546.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
38694.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
38694.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
38740.jpg Is this artwork displayed in musée toulouse-lautrec, albi? Please answer yes or no. Yes
38740.jpg Is this artwork displayed in kupferstichkabinett, gotha? Please answer yes or no. No
38881.jpg Does this artwork belong to the type of genre? Please answer yes or no. Yes
38881.jpg Does this artwork belong to the type of religious? Please answer yes or no. No
38993.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
38993.jpg Does this artwork exist in the form of illumination? Please answer yes or no. No
39026.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
39026.jpg Does this artwork belong to the type of historical? Please answer yes or no. No
39124.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
39124.jpg Does this artwork exist in the form of graphics? Please answer yes or no. No
39188.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
39188.jpg Does this artwork exist in the form of architecture? Please answer yes or no. No
39482.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
39482.jpg Does this artwork exist in the form of metalwork? Please answer yes or no. No
39556.jpg Is this artwork created by unknown master, dutch? Please answer yes or no. Yes
39556.jpg Is this artwork created by cuyp, benjamin gerritsz.? Please answer yes or no. No
41036.jpg Is this artwork displayed in kunsthistorisches museum, vienna? Please answer yes or no. Yes
41036.jpg Is this artwork displayed in national museum of art, minsk? Please answer yes or no. No
41371.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
41371.jpg Does this artwork exist in the form of architecture? Please answer yes or no. No
41484.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
41484.jpg Does this artwork belong to the type of historical? Please answer yes or no. No
41594.jpg Is this artwork created by veronese, paolo? Please answer yes or no. Yes
41594.jpg Is this artwork created by jeaurat, etienne? Please answer yes or no. No
416.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes
416.jpg Does this artwork exist in the form of others? Please answer yes or no. No
41653.jpg Is this artwork titled view of the sala del collegio? Please answer yes or no. Yes
41653.jpg Is this artwork titled reine lefebvre and margot? Please answer yes or no. No
41944.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
41944.jpg Does this artwork exist in the form of mosaic? Please answer yes or no. No
42152.jpg Is this artwork titled the pieterskerk in leiden? Please answer yes or no. Yes
42152.jpg Is this artwork titled portrait of cardinal reginald pole? Please answer yes or no. No
42288.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
42288.jpg Does this artwork exist in the form of stained-glass? Please answer yes or no. No
42303.jpg Is this artwork displayed in art museum, cincinnati? Please answer yes or no. Yes
42303.jpg Is this artwork displayed in banca del monte di bologna e ravenna, bologna? Please answer yes or no. No
42401.jpg Is this artwork created by waldmüller, fedinand georg? Please answer yes or no. Yes
42401.jpg Is this artwork created by seeman, enoch? Please answer yes or no. No
42447.jpg Is this artwork displayed in musée du louvre, paris? Please answer yes or no. Yes
42447.jpg Is this artwork displayed in santa catarina, pisa? Please answer yes or no. No
42585.jpg Is this artwork created by werff, pieter van der? Please answer yes or no. Yes
42585.jpg Is this artwork created by domenichini, apollonio? Please answer yes or no. No
42706.jpg Is this artwork displayed in musée du louvre, paris? Please answer yes or no. Yes
42706.jpg Is this artwork displayed in galleria nazionale d'arte moderna e contemporanea, rome? Please answer yes or no. No
42796.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
42796.jpg Is this artwork displayed in museo di san salvi, florence? Please answer yes or no. No
42857.jpg Does this artwork belong to the type of landscape? Please answer yes or no. Yes
42857.jpg Does this artwork belong to the type of study? Please answer yes or no. No
42905.jpg Is this artwork created by wit, jacob de? Please answer yes or no. Yes
42905.jpg Is this artwork created by vittone, bernardo antonio? Please answer yes or no. No
42941.jpg Is this artwork created by witte, emanuel de? Please answer yes or no. Yes
42941.jpg Is this artwork created by bicci di neri? Please answer yes or no. No
42956.jpg Is this artwork titled view of rome with the tiberand castel sant'angelo? Please answer yes or no. Yes
42956.jpg Is this artwork titled st bonaventure enters the franciscan order? Please answer yes or no. No
42987.jpg Is this artwork created by witz, konrad? Please answer yes or no. Yes
42987.jpg Is this artwork created by christus, petrus? Please answer yes or no. No
43142.jpg Does this artwork belong to the type of mythological? Please answer yes or no. Yes
43142.jpg Does this artwork belong to the type of interior? Please answer yes or no. No
43175.jpg Is this artwork displayed in private collection? Please answer yes or no. Yes
43175.jpg Is this artwork displayed in smith college museum of art, northampton? Please answer yes or no. No
43349.jpg Is this artwork created by zuccarelli, francesco? Please answer yes or no. Yes
43349.jpg Is this artwork created by baccanelli, giovanni antonio di giulio? Please answer yes or no. No
43445.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
43445.jpg Does this artwork belong to the type of interior? Please answer yes or no. No
4836.jpg Is this artwork displayed in villa cornaro, piombino dese? Please answer yes or no. Yes
4836.jpg Is this artwork displayed in palais saint-vaast, arras? Please answer yes or no. No
5227.jpg Is this artwork created by botticelli, sandro? Please answer yes or no. Yes
5227.jpg Is this artwork created by vigri, caterina? Please answer yes or no. No
526.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
526.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No
5906.jpg Is this artwork created by bronzino, agnolo? Please answer yes or no. Yes
5906.jpg Is this artwork created by pellegrino da san daniele? Please answer yes or no. No
6168.jpg Does this artwork exist in the form of graphics? Please answer yes or no. Yes
6168.jpg Does this artwork exist in the form of tapestry? Please answer yes or no. No
6297.jpg Is this artwork titled peasants making merry outside a tavern 'the swan'? Please answer yes or no. Yes
6297.jpg Is this artwork titled allegory of quietude? Please answer yes or no. No
6478.jpg Does this artwork belong to the type of religious? Please answer yes or no. Yes
6478.jpg Does this artwork belong to the type of genre? Please answer yes or no. No
6969.jpg Is this artwork titled letizia ramolino bonaparte? Please answer yes or no. Yes
6969.jpg Is this artwork titled job and his daughters? Please answer yes or no. No
701.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
701.jpg Does this artwork exist in the form of others? Please answer yes or no. No
7702.jpg Is this artwork titled reine lefebvre and margot? Please answer yes or no. Yes
7702.jpg Is this artwork titled fire in the oil depot at san marcuola? Please answer yes or no. No
8101.jpg Is this artwork displayed in museu de arte, são paulo? Please answer yes or no. Yes
8101.jpg Is this artwork displayed in national széchényi library, budapest? Please answer yes or no. No
815.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
815.jpg Does this artwork exist in the form of furniture? Please answer yes or no. No
8797.jpg Is this artwork created by coecke van aelst, pieter? Please answer yes or no. Yes
8797.jpg Is this artwork created by abaquesne, masséot? Please answer yes or no. No
8885.jpg Is this artwork displayed in art museum, saint louis? Please answer yes or no. Yes
8885.jpg Is this artwork displayed in museo civico d'arte antica, turin? Please answer yes or no. No
9153.jpg Is this artwork displayed in galleria nazionale, parma? Please answer yes or no. Yes
9153.jpg Is this artwork displayed in hospital de san bernardo, seville? Please answer yes or no. No
9395.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
9395.jpg Does this artwork exist in the form of stained-glass? Please answer yes or no. No
9405.jpg Is this artwork created by courbet, gustave? Please answer yes or no. Yes
9405.jpg Is this artwork created by milani, aureliano? Please answer yes or no. No
9599.jpg Does this artwork exist in the form of painting? Please answer yes or no. Yes
9599.jpg Does this artwork exist in the form of ceramics? Please answer yes or no. No
995.jpg Does this artwork exist in the form of sculpture? Please answer yes or no. Yes
995.jpg Does this artwork exist in the form of painting? Please answer yes or no. No
internvl_chat/eval/mme/Your_Results/celebrity.txt 0 → 100644
View file @ 26e59280
tt0032138_shot_0395_img_0.jpg Is the actor inside the red bounding box named Frank Morgan? Please answer yes or no. Yes
tt0032138_shot_0395_img_0.jpg Is the actor inside the red bounding box named Eric Schniewind? Please answer yes or no. No
tt0035423_shot_0464_img_0.jpg Is the actor inside the red bounding box called Hugh Jackman? Please answer yes or no. Yes
tt0035423_shot_0464_img_0.jpg Is the actor inside the red bounding box called Lizzie Hopley? Please answer yes or no. No
tt0038650_shot_0737_img_1.jpg Is the person inside the red bounding box called James Stewart? Please answer yes or no. Yes
tt0038650_shot_0737_img_1.jpg Is the person inside the red bounding box called Phil Selway? Please answer yes or no. No
tt0047396_shot_0333_img_0.jpg Is the actor inside the red bounding box named James Stewart? Please answer yes or no. Yes
tt0047396_shot_0333_img_0.jpg Is the actor inside the red bounding box named Ron Blair? Please answer yes or no. No
tt0048545_shot_0124_img_0.jpg Is the actor inside the red bounding box called Natalie Wood? Please answer yes or no. Yes
tt0048545_shot_0124_img_0.jpg Is the actor inside the red bounding box called Rebecca Jackson Mendoza? Please answer yes or no. No
tt0049470_shot_0279_img_0.jpg Is the person inside the red bounding box called James Stewart? Please answer yes or no. Yes
tt0049470_shot_0279_img_0.jpg Is the person inside the red bounding box called Matt Pashkow? Please answer yes or no. No
tt0049730_shot_0273_img_0.jpg Is the person inside the red bounding box called Vera Miles? Please answer yes or no. Yes
tt0049730_shot_0273_img_0.jpg Is the person inside the red bounding box called Addie Yungmee? Please answer yes or no. No
tt0052357_shot_0511_img_0.jpg Is the actor inside the red bounding box called Kim Novak? Please answer yes or no. Yes
tt0052357_shot_0511_img_0.jpg Is the actor inside the red bounding box called Abigail Van Alyn? Please answer yes or no. No
tt0053221_shot_0197_img_0.jpg Is the actor inside the red bounding box named John Wayne? Please answer yes or no. Yes
tt0053221_shot_0197_img_0.jpg Is the actor inside the red bounding box named Claude-Oliver Rudolph? Please answer yes or no. No
tt0054167_shot_0122_img_0.jpg Is the person inside the red bounding box called Anna Massey? Please answer yes or no. Yes
tt0054167_shot_0122_img_0.jpg Is the person inside the red bounding box called Eddie Tagoe? Please answer yes or no. No
tt0056869_shot_0320_img_0.jpg Is the person inside the red bounding box called Tippi Hedren? Please answer yes or no. Yes
tt0056869_shot_0320_img_0.jpg Is the person inside the red bounding box called Denise Mack? Please answer yes or no. No
tt0056923_shot_0835_img_0.jpg Is the actor inside the red bounding box called Audrey Hepburn? Please answer yes or no. Yes
tt0056923_shot_0835_img_0.jpg Is the actor inside the red bounding box called Chris April? Please answer yes or no. No
tt0057115_shot_0686_img_0.jpg Is the person inside the red bounding box named James Garner? Please answer yes or no. Yes
tt0057115_shot_0686_img_0.jpg Is the person inside the red bounding box named Chutimon Chuengcharoensukying? Please answer yes or no. No
tt0058331_shot_0353_img_0.jpg Is the actor inside the red bounding box named Julie Andrews? Please answer yes or no. Yes
tt0058331_shot_0353_img_0.jpg Is the actor inside the red bounding box named Ed Geldart? Please answer yes or no. No
tt0058461_shot_0901_img_0.jpg Is the actor inside the red bounding box called Gian Maria Volontè? Please answer yes or no. Yes
tt0058461_shot_0901_img_0.jpg Is the actor inside the red bounding box called Jennifer Connelly? Please answer yes or no. No
tt0061418_shot_0148_img_0.jpg Is the actor inside the red bounding box named Faye Dunaway? Please answer yes or no. Yes
tt0061418_shot_0148_img_0.jpg Is the actor inside the red bounding box named Warona Seane? Please answer yes or no. No
tt0061722_shot_0259_img_0.jpg Is the actor inside the red bounding box called Dustin Hoffman? Please answer yes or no. Yes
tt0061722_shot_0259_img_0.jpg Is the actor inside the red bounding box called Christopher Olsen? Please answer yes or no. No
tt0062622_shot_0291_img_0.jpg Is the actor inside the red bounding box named Keir Dullea? Please answer yes or no. Yes
tt0062622_shot_0291_img_0.jpg Is the actor inside the red bounding box named Frank Albanese? Please answer yes or no. No
tt0063442_shot_0702_img_0.jpg Is the actor inside the red bounding box called Linda Harrison? Please answer yes or no. Yes
tt0063442_shot_0702_img_0.jpg Is the actor inside the red bounding box called Michael McKean? Please answer yes or no. No
tt0064115_shot_0367_img_0.jpg Is the actor inside the red bounding box named Robert Redford? Please answer yes or no. Yes
tt0064115_shot_0367_img_0.jpg Is the actor inside the red bounding box named Cooper Murray? Please answer yes or no. No
tt0064665_shot_0300_img_0.jpg Is the actor inside the red bounding box called Jon Voight? Please answer yes or no. Yes
tt0064665_shot_0300_img_0.jpg Is the actor inside the red bounding box called Harvey Meyer? Please answer yes or no. No
tt0065214_shot_0366_img_0.jpg Is the person inside the red bounding box called Robert Ryan? Please answer yes or no. Yes
tt0065214_shot_0366_img_0.jpg Is the person inside the red bounding box called Victor Verhaeghe? Please answer yes or no. No
tt0065724_shot_0320_img_1.jpg Is the person inside the red bounding box named Karen Black? Please answer yes or no. Yes
tt0065724_shot_0320_img_1.jpg Is the person inside the red bounding box named Nick Discenza? Please answer yes or no. No
tt0066026_shot_0085_img_0.jpg Is the person inside the red bounding box called Donald Sutherland? Please answer yes or no. Yes
tt0066026_shot_0085_img_0.jpg Is the person inside the red bounding box called Michael Wollet? Please answer yes or no. No
tt0066921_shot_0631_img_0.jpg Is the actor inside the red bounding box called Malcolm McDowell? Please answer yes or no. Yes
tt0066921_shot_0631_img_0.jpg Is the actor inside the red bounding box called Darling Légitimus? Please answer yes or no. No
tt0067116_shot_0122_img_0.jpg Is the actor inside the red bounding box called Gene Hackman? Please answer yes or no. Yes
tt0067116_shot_0122_img_0.jpg Is the actor inside the red bounding box called Russell G. Jones? Please answer yes or no. No
tt0068646_shot_0166_img_0.jpg Is the actor inside the red bounding box called Marlon Brando? Please answer yes or no. Yes
tt0068646_shot_0166_img_0.jpg Is the actor inside the red bounding box called Voltaire Sterling? Please answer yes or no. No
tt0069762_shot_0723_img_0.jpg Is the person inside the red bounding box named Sissy Spacek? Please answer yes or no. Yes
tt0069762_shot_0723_img_0.jpg Is the person inside the red bounding box named Monica Giordano? Please answer yes or no. No
tt0070047_shot_0255_img_0.jpg Is the actor inside the red bounding box called Ellen Burstyn? Please answer yes or no. Yes
tt0070047_shot_0255_img_0.jpg Is the actor inside the red bounding box called Shawnee Smith? Please answer yes or no. No
tt0070379_shot_0569_img_0.jpg Is the actor inside the red bounding box named Richard Romanus? Please answer yes or no. Yes
tt0070379_shot_0569_img_0.jpg Is the actor inside the red bounding box named Valerie Colgan? Please answer yes or no. No
tt0070511_shot_0639_img_0.jpg Is the person inside the red bounding box called Dustin Hoffman? Please answer yes or no. Yes
tt0070511_shot_0639_img_0.jpg Is the person inside the red bounding box called Fernando Lueches? Please answer yes or no. No
tt0070735_shot_0818_img_0.jpg Is the person inside the red bounding box named Robert Redford? Please answer yes or no. Yes
tt0070735_shot_0818_img_0.jpg Is the person inside the red bounding box named Ellin Dennis? Please answer yes or no. No
tt0070849_shot_0021_img_1.jpg Is the person inside the red bounding box named Maria Schneider? Please answer yes or no. Yes
tt0070849_shot_0021_img_1.jpg Is the person inside the red bounding box named Mary Kellogg? Please answer yes or no. No
tt0071315_shot_0153_img_0.jpg Is the actor inside the red bounding box named Faye Dunaway? Please answer yes or no. Yes
tt0071315_shot_0153_img_0.jpg Is the actor inside the red bounding box named Kelly Hitman? Please answer yes or no. No
tt0071562_shot_0684_img_0.jpg Is the actor inside the red bounding box named Al Pacino? Please answer yes or no. Yes
tt0071562_shot_0684_img_0.jpg Is the actor inside the red bounding box named Debie Jarczewski? Please answer yes or no. No
tt0072684_shot_0512_img_1.jpg Is the person inside the red bounding box named Marisa Berenson? Please answer yes or no. Yes
tt0072684_shot_0512_img_1.jpg Is the person inside the red bounding box named Graham Bohea? Please answer yes or no. No
tt0073195_shot_0280_img_0.jpg Is the actor inside the red bounding box named Roy Scheider? Please answer yes or no. Yes
tt0073195_shot_0280_img_0.jpg Is the actor inside the red bounding box named Abdul Qadir Farookh? Please answer yes or no. No
tt0073629_shot_0700_img_0.jpg Is the person inside the red bounding box named Barry Bostwick? Please answer yes or no. Yes
tt0073629_shot_0700_img_0.jpg Is the person inside the red bounding box named Johnny Galecki? Please answer yes or no. No
tt0074119_shot_0814_img_0.jpg Is the actor inside the red bounding box called Robert Redford? Please answer yes or no. Yes
tt0074119_shot_0814_img_0.jpg Is the actor inside the red bounding box called Delroy Lindo? Please answer yes or no. No
tt0074285_shot_0535_img_1.jpg Is the person inside the red bounding box named William Katt? Please answer yes or no. Yes
tt0074285_shot_0535_img_1.jpg Is the person inside the red bounding box named Stephen Rider? Please answer yes or no. No
tt0075148_shot_0618_img_0.jpg Is the actor inside the red bounding box called Sylvester Stallone? Please answer yes or no. Yes
tt0075148_shot_0618_img_0.jpg Is the actor inside the red bounding box called Eric Hatch? Please answer yes or no. No
tt0075686_shot_0373_img_0.jpg Is the actor inside the red bounding box called Woody Allen? Please answer yes or no. Yes
tt0075686_shot_0373_img_0.jpg Is the actor inside the red bounding box called Penny Wallace? Please answer yes or no. No
tt0076729_shot_0451_img_0.jpg Is the actor inside the red bounding box called Sally Field? Please answer yes or no. Yes
tt0076729_shot_0451_img_0.jpg Is the actor inside the red bounding box called Giorgio Libassi? Please answer yes or no. No
tt0076759_shot_0930_img_0.jpg Is the actor inside the red bounding box called Harrison Ford? Please answer yes or no. Yes
tt0076759_shot_0930_img_0.jpg Is the actor inside the red bounding box called Ryoko Sadoshima? Please answer yes or no. No
tt0077402_shot_1220_img_0.jpg Is the person inside the red bounding box named Scott H. Reiniger? Please answer yes or no. Yes
tt0077402_shot_1220_img_0.jpg Is the person inside the red bounding box named Chris Delaney? Please answer yes or no. No
tt0077405_shot_0150_img_0.jpg Is the actor inside the red bounding box named Sam Shepard? Please answer yes or no. Yes
tt0077405_shot_0150_img_0.jpg Is the actor inside the red bounding box named Bijou Phillips? Please answer yes or no. No
tt0077416_shot_1442_img_0.jpg Is the person inside the red bounding box named Robert De Niro? Please answer yes or no. Yes
tt0077416_shot_1442_img_0.jpg Is the person inside the red bounding box named Stu Smith? Please answer yes or no. No
tt0077651_shot_0133_img_0.jpg Is the person inside the red bounding box called Jamie Lee Curtis? Please answer yes or no. Yes
tt0077651_shot_0133_img_0.jpg Is the person inside the red bounding box called Paris Arrowsmith? Please answer yes or no. No
tt0078788_shot_1434_img_0.jpg Is the person inside the red bounding box called Martin Sheen? Please answer yes or no. Yes
tt0078788_shot_1434_img_0.jpg Is the person inside the red bounding box called Le Capriccio Français? Please answer yes or no. No
tt0078841_shot_0692_img_0.jpg Is the actor inside the red bounding box named Shirley MacLaine? Please answer yes or no. Yes
tt0078841_shot_0692_img_0.jpg Is the actor inside the red bounding box named Tomas Choy? Please answer yes or no. No
tt0079417_shot_0735_img_0.jpg Is the actor inside the red bounding box called Meryl Streep? Please answer yes or no. Yes
tt0079417_shot_0735_img_0.jpg Is the actor inside the red bounding box called Ross Lacy? Please answer yes or no. No
tt0079470_shot_0798_img_0.jpg Is the person inside the red bounding box named Eric Idle? Please answer yes or no. Yes
tt0079470_shot_0798_img_0.jpg Is the person inside the red bounding box named Quincy Taylor? Please answer yes or no. No
tt0079945_shot_1411_img_0.jpg Is the person inside the red bounding box named Persis Khambatta? Please answer yes or no. Yes
tt0079945_shot_1411_img_0.jpg Is the person inside the red bounding box named Alison Waddell? Please answer yes or no. No
tt0080339_shot_0711_img_0.jpg Is the actor inside the red bounding box named Robert Hays? Please answer yes or no. Yes
tt0080339_shot_0711_img_0.jpg Is the actor inside the red bounding box named Grace Sullivan? Please answer yes or no. No
tt0080684_shot_1574_img_2.jpg Is the actor inside the red bounding box called Mark Hamill? Please answer yes or no. Yes
tt0080684_shot_1574_img_2.jpg Is the actor inside the red bounding box called Rodion Salnikov? Please answer yes or no. No
tt0081505_shot_0449_img_0.jpg Is the actor inside the red bounding box called Shelley Duvall? Please answer yes or no. Yes
tt0081505_shot_0449_img_0.jpg Is the actor inside the red bounding box called Antony Carrick? Please answer yes or no. No
tt0082089_shot_0046_img_0.jpg Is the actor inside the red bounding box named Kathleen Turner? Please answer yes or no. Yes
tt0082089_shot_0046_img_0.jpg Is the actor inside the red bounding box named Aaron Henderson? Please answer yes or no. No
tt0082198_shot_1353_img_0.jpg Is the person inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes
tt0082198_shot_1353_img_0.jpg Is the person inside the red bounding box named Tim Herlihy? Please answer yes or no. No
tt0082971_shot_0831_img_0.jpg Is the actor inside the red bounding box called Harrison Ford? Please answer yes or no. Yes
tt0082971_shot_0831_img_0.jpg Is the actor inside the red bounding box called Richard Angarola? Please answer yes or no. No
tt0083658_shot_0963_img_0.jpg Is the person inside the red bounding box called Rutger Hauer? Please answer yes or no. Yes
tt0083658_shot_0963_img_0.jpg Is the person inside the red bounding box called Stéphane Julien? Please answer yes or no. No
tt0083866_shot_0364_img_0.jpg Is the actor inside the red bounding box called Robert MacNaughton? Please answer yes or no. Yes
tt0083866_shot_0364_img_0.jpg Is the actor inside the red bounding box called Seam Turay? Please answer yes or no. No
tt0083907_shot_0633_img_0.jpg Is the actor inside the red bounding box named Bruce Campbell? Please answer yes or no. Yes
tt0083907_shot_0633_img_0.jpg Is the actor inside the red bounding box named Kaden Leos? Please answer yes or no. No
tt0083929_shot_0405_img_0.jpg Is the actor inside the red bounding box named Jennifer Jason Leigh? Please answer yes or no. Yes
tt0083929_shot_0405_img_0.jpg Is the actor inside the red bounding box named Eric D. Sandgren? Please answer yes or no. No
tt0084726_shot_0283_img_0.jpg Is the actor inside the red bounding box named Leonard Nimoy? Please answer yes or no. Yes
tt0084726_shot_0283_img_0.jpg Is the actor inside the red bounding box named John Cusack? Please answer yes or no. No
tt0086190_shot_0815_img_0.jpg Is the actor inside the red bounding box named Carrie Fisher? Please answer yes or no. Yes
tt0086190_shot_0815_img_0.jpg Is the actor inside the red bounding box named Ernie Adams? Please answer yes or no. No
tt0086250_shot_1079_img_0.jpg Is the actor inside the red bounding box called Steven Bauer? Please answer yes or no. Yes
tt0086250_shot_1079_img_0.jpg Is the actor inside the red bounding box called Bill Nunn? Please answer yes or no. No
tt0086856_shot_0929_img_0.jpg Is the actor inside the red bounding box called Peter Weller? Please answer yes or no. Yes
tt0086856_shot_0929_img_0.jpg Is the actor inside the red bounding box called Tracee Cocco? Please answer yes or no. No
tt0086879_shot_0158_img_0.jpg Is the person inside the red bounding box called Elizabeth Berridge? Please answer yes or no. Yes
tt0086879_shot_0158_img_0.jpg Is the person inside the red bounding box called Ralph Ineson? Please answer yes or no. No
tt0087332_shot_0798_img_0.jpg Is the person inside the red bounding box called Bill Murray? Please answer yes or no. Yes
tt0087332_shot_0798_img_0.jpg Is the person inside the red bounding box called Jiao Xu? Please answer yes or no. No
tt0087469_shot_0049_img_2.jpg Is the person inside the red bounding box named Harrison Ford? Please answer yes or no. Yes
tt0087469_shot_0049_img_2.jpg Is the person inside the red bounding box named Paulo Benedeti? Please answer yes or no. No
tt0088847_shot_0109_img_0.jpg Is the actor inside the red bounding box named Anthony Michael Hall? Please answer yes or no. Yes
tt0088847_shot_0109_img_0.jpg Is the actor inside the red bounding box named Luis Javier? Please answer yes or no. No
tt0088944_shot_0634_img_0.jpg Is the actor inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes
tt0088944_shot_0634_img_0.jpg Is the actor inside the red bounding box named Shaine Jones? Please answer yes or no. No
tt0088993_shot_0569_img_0.jpg Is the actor inside the red bounding box called George A. Romero? Please answer yes or no. Yes
tt0088993_shot_0569_img_0.jpg Is the actor inside the red bounding box called James Eckhouse? Please answer yes or no. No
tt0089218_shot_0327_img_0.jpg Is the person inside the red bounding box named Sean Astin? Please answer yes or no. Yes
tt0089218_shot_0327_img_0.jpg Is the person inside the red bounding box named Dan Hunter? Please answer yes or no. No
tt0089881_shot_0034_img_0.jpg Is the actor inside the red bounding box called Tatsuya Nakadai? Please answer yes or no. Yes
tt0089881_shot_0034_img_0.jpg Is the actor inside the red bounding box called Nancy Vee? Please answer yes or no. No
tt0090022_shot_0464_img_0.jpg Is the actor inside the red bounding box called Scott Glenn? Please answer yes or no. Yes
tt0090022_shot_0464_img_0.jpg Is the actor inside the red bounding box called Robert Ryan? Please answer yes or no. No
tt0090605_shot_0344_img_0.jpg Is the person inside the red bounding box called Sigourney Weaver? Please answer yes or no. Yes
tt0090605_shot_0344_img_0.jpg Is the person inside the red bounding box called Lia Beldam? Please answer yes or no. No
tt0090756_shot_0135_img_0.jpg Is the person inside the red bounding box named Laura Dern? Please answer yes or no. Yes
tt0090756_shot_0135_img_0.jpg Is the person inside the red bounding box named Keith Frost? Please answer yes or no. No
tt0091042_shot_0098_img_0.jpg Is the person inside the red bounding box called Matthew Broderick? Please answer yes or no. Yes
tt0091042_shot_0098_img_0.jpg Is the person inside the red bounding box called Mina E. Mina? Please answer yes or no. No
tt0091738_shot_0073_img_1.jpg Is the actor inside the red bounding box called Kathleen Turner? Please answer yes or no. Yes
tt0091738_shot_0073_img_1.jpg Is the actor inside the red bounding box called Pat Kiernan? Please answer yes or no. No
tt0091867_shot_0422_img_2.jpg Is the person inside the red bounding box named Simon Callow? Please answer yes or no. Yes
tt0091867_shot_0422_img_2.jpg Is the person inside the red bounding box named Rusty Goffe? Please answer yes or no. No
tt0092099_shot_0455_img_1.jpg Is the person inside the red bounding box called Tom Cruise? Please answer yes or no. Yes
tt0092099_shot_0455_img_1.jpg Is the person inside the red bounding box called Carol Krolick? Please answer yes or no. No
tt0092699_shot_0208_img_0.jpg Is the actor inside the red bounding box called William Hurt? Please answer yes or no. Yes
tt0092699_shot_0208_img_0.jpg Is the actor inside the red bounding box called Hildur Ruriks? Please answer yes or no. No
tt0093565_shot_0409_img_0.jpg Is the actor inside the red bounding box named Cher? Please answer yes or no. Yes
tt0093565_shot_0409_img_0.jpg Is the actor inside the red bounding box named Mark Brady? Please answer yes or no. No
tt0093748_shot_0346_img_0.jpg Is the actor inside the red bounding box called John Candy? Please answer yes or no. Yes
tt0093748_shot_0346_img_0.jpg Is the actor inside the red bounding box called Sarah Heller? Please answer yes or no. No
tt0093773_shot_0212_img_0.jpg Is the person inside the red bounding box named Jesse Ventura? Please answer yes or no. Yes
tt0093773_shot_0212_img_0.jpg Is the person inside the red bounding box named Akio Mitamura? Please answer yes or no. No
tt0093779_shot_1047_img_0.jpg Is the person inside the red bounding box named Peter Falk? Please answer yes or no. Yes
tt0093779_shot_1047_img_0.jpg Is the person inside the red bounding box named Lisa Ann Walter? Please answer yes or no. No
tt0094226_shot_0237_img_2.jpg Is the actor inside the red bounding box called Kevin Costner? Please answer yes or no. Yes
tt0094226_shot_0237_img_2.jpg Is the actor inside the red bounding box called Colin Hill? Please answer yes or no. No
tt0094737_shot_0567_img_0.jpg Is the person inside the red bounding box called Tom Hanks? Please answer yes or no. Yes
tt0094737_shot_0567_img_0.jpg Is the person inside the red bounding box called Chris McHallem? Please answer yes or no. No
tt0095016_shot_1170_img_0.jpg Is the actor inside the red bounding box called Paul Gleason? Please answer yes or no. Yes
tt0095016_shot_1170_img_0.jpg Is the actor inside the red bounding box called Carl Palmer? Please answer yes or no. No
tt0095250_shot_0509_img_0.jpg Is the actor inside the red bounding box named Jean Reno? Please answer yes or no. Yes
tt0095250_shot_0509_img_0.jpg Is the actor inside the red bounding box named Ralph Meyering Jr.? Please answer yes or no. No
tt0095765_shot_0008_img_0.jpg Is the actor inside the red bounding box called Antonella Attili? Please answer yes or no. Yes
tt0095765_shot_0008_img_0.jpg Is the actor inside the red bounding box called Amber Estrada? Please answer yes or no. No
tt0095953_shot_0412_img_0.jpg Is the person inside the red bounding box named Tom Cruise? Please answer yes or no. Yes
tt0095953_shot_0412_img_0.jpg Is the person inside the red bounding box named Lara Mulcahy? Please answer yes or no. No
tt0096320_shot_0085_img_0.jpg Is the actor inside the red bounding box called Arnold Schwarzenegger? Please answer yes or no. Yes
tt0096320_shot_0085_img_0.jpg Is the actor inside the red bounding box called Dan Duran? Please answer yes or no. No
tt0096754_shot_0570_img_1.jpg Is the person inside the red bounding box named Todd Graff? Please answer yes or no. Yes
tt0096754_shot_0570_img_1.jpg Is the person inside the red bounding box named Guy Carleton? Please answer yes or no. No
tt0096874_shot_0647_img_0.jpg Is the actor inside the red bounding box named Michael J. Fox? Please answer yes or no. Yes
tt0096874_shot_0647_img_0.jpg Is the actor inside the red bounding box named Momoko Komatsu? Please answer yes or no. No
tt0096895_shot_0819_img_1.jpg Is the person inside the red bounding box called Michael Keaton? Please answer yes or no. Yes
tt0096895_shot_0819_img_1.jpg Is the person inside the red bounding box called Ben Foster? Please answer yes or no. No
tt0097216_shot_0381_img_0.jpg Is the actor inside the red bounding box named Danny Aiello? Please answer yes or no. Yes
tt0097216_shot_0381_img_0.jpg Is the actor inside the red bounding box named Taissa Farmiga? Please answer yes or no. No
tt0097428_shot_0106_img_0.jpg Is the actor inside the red bounding box named Bill Murray? Please answer yes or no. Yes
tt0097428_shot_0106_img_0.jpg Is the actor inside the red bounding box named Michael Fawcett? Please answer yes or no. No
tt0097576_shot_1010_img_2.jpg Is the actor inside the red bounding box named Harrison Ford? Please answer yes or no. Yes
tt0097576_shot_1010_img_2.jpg Is the actor inside the red bounding box named M. Emmet Walsh? Please answer yes or no. No
tt0098635_shot_0556_img_0.jpg Is the actor inside the red bounding box named Meg Ryan? Please answer yes or no. Yes
tt0098635_shot_0556_img_0.jpg Is the actor inside the red bounding box named Tom Branch? Please answer yes or no. No
tt0098724_shot_0474_img_0.jpg Is the person inside the red bounding box named Andie MacDowell? Please answer yes or no. Yes
tt0098724_shot_0474_img_0.jpg Is the person inside the red bounding box named Linda Taylor? Please answer yes or no. No
tt0099423_shot_1010_img_0.jpg Is the person inside the red bounding box called Bruce Willis? Please answer yes or no. Yes
tt0099423_shot_1010_img_0.jpg Is the person inside the red bounding box called Trevor Eve? Please answer yes or no. No
tt0099487_shot_0123_img_0.jpg Is the actor inside the red bounding box named Johnny Depp? Please answer yes or no. Yes
tt0099487_shot_0123_img_0.jpg Is the actor inside the red bounding box named Farrah Forke? Please answer yes or no. No
tt0099674_shot_1356_img_0.jpg Is the person inside the red bounding box named Al Pacino? Please answer yes or no. Yes
tt0099674_shot_1356_img_0.jpg Is the person inside the red bounding box named Nick Porrazzo? Please answer yes or no. No
tt0099685_shot_1132_img_0.jpg Is the actor inside the red bounding box called Ray Liotta? Please answer yes or no. Yes
tt0099685_shot_1132_img_0.jpg Is the actor inside the red bounding box called Chick Allan? Please answer yes or no. No
tt0099810_shot_0285_img_0.jpg Is the person inside the red bounding box called Alec Baldwin? Please answer yes or no. Yes
tt0099810_shot_0285_img_0.jpg Is the person inside the red bounding box called Jennifer Anglin? Please answer yes or no. No
tt0100157_shot_0365_img_0.jpg Is the actor inside the red bounding box named James Caan? Please answer yes or no. Yes
tt0100157_shot_0365_img_0.jpg Is the actor inside the red bounding box named Bryan Johnson? Please answer yes or no. No
tt0100403_shot_0517_img_0.jpg Is the person inside the red bounding box called Gary Busey? Please answer yes or no. Yes
tt0100403_shot_0517_img_0.jpg Is the person inside the red bounding box called Alfred Tiaki Hotu? Please answer yes or no. No
tt0100405_shot_0786_img_0.jpg Is the actor inside the red bounding box named Jason Alexander? Please answer yes or no. Yes
tt0100405_shot_0786_img_0.jpg Is the actor inside the red bounding box named Alexandra Bastedo? Please answer yes or no. No
tt0101410_shot_0105_img_0.jpg Is the person inside the red bounding box named John Turturro? Please answer yes or no. Yes
tt0101410_shot_0105_img_0.jpg Is the person inside the red bounding box named David Gore? Please answer yes or no. No
tt0102492_shot_0086_img_0.jpg Is the actor inside the red bounding box called Jamie Lee Curtis? Please answer yes or no. Yes
tt0102492_shot_0086_img_0.jpg Is the actor inside the red bounding box called Heidi Fischer? Please answer yes or no. No
tt0103064_shot_1206_img_0.jpg Is the actor inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes
tt0103064_shot_1206_img_0.jpg Is the actor inside the red bounding box named Gigi Lee? Please answer yes or no. No
tt0103064_shot_2602_img_1.jpg Is the person inside the red bounding box named Arnold Schwarzenegger? Please answer yes or no. Yes
tt0103064_shot_2602_img_1.jpg Is the person inside the red bounding box named Candice Azzara? Please answer yes or no. No
tt0103776_shot_0719_img_0.jpg Is the person inside the red bounding box called Michael Keaton? Please answer yes or no. Yes
tt0103776_shot_0719_img_0.jpg Is the person inside the red bounding box called Nicholas Rice? Please answer yes or no. No
tt0104036_shot_0336_img_1.jpg Is the person inside the red bounding box named Stephen Rea? Please answer yes or no. Yes
tt0104036_shot_0336_img_1.jpg Is the person inside the red bounding box named Mimi Lizio? Please answer yes or no. No
tt0104257_shot_0477_img_0.jpg Is the person inside the red bounding box named Jack Nicholson? Please answer yes or no. Yes
tt0104257_shot_0477_img_0.jpg Is the person inside the red bounding box named Emma Julia Jacobs? Please answer yes or no. No
tt0104348_shot_0340_img_0.jpg Is the person inside the red bounding box called Ed Harris? Please answer yes or no. Yes
tt0104348_shot_0340_img_0.jpg Is the person inside the red bounding box called Carla Lizzette Mejia? Please answer yes or no. No
tt0105236_shot_0193_img_0.jpg Is the actor inside the red bounding box named Harvey Keitel? Please answer yes or no. Yes
tt0105236_shot_0193_img_0.jpg Is the actor inside the red bounding box named Terence Yin? Please answer yes or no. No
tt0105665_shot_0351_img_0.jpg Is the actor inside the red bounding box named Kyle MacLachlan? Please answer yes or no. Yes
tt0105665_shot_0351_img_0.jpg Is the actor inside the red bounding box named Julia Hsu? Please answer yes or no. No
tt0105695_shot_1436_img_1.jpg Is the person inside the red bounding box called Jaimz Woolvett? Please answer yes or no. Yes
tt0105695_shot_1436_img_1.jpg Is the person inside the red bounding box called Hermione Baddeley? Please answer yes or no. No
tt0106977_shot_1604_img_0.jpg Is the person inside the red bounding box named Tommy Lee Jones? Please answer yes or no. Yes
tt0106977_shot_1604_img_0.jpg Is the person inside the red bounding box named Honey Chhaya? Please answer yes or no. No
tt0107614_shot_0116_img_0.jpg Is the person inside the red bounding box called Sally Field? Please answer yes or no. Yes
tt0107614_shot_0116_img_0.jpg Is the person inside the red bounding box called Arthur Senzy? Please answer yes or no. No
tt0108399_shot_0778_img_0.jpg Is the actor inside the red bounding box called Christopher Walken? Please answer yes or no. Yes
tt0108399_shot_0778_img_0.jpg Is the actor inside the red bounding box called Fiona Sit? Please answer yes or no. No
tt0109831_shot_0298_img_0.jpg Is the person inside the red bounding box called Hugh Grant? Please answer yes or no. Yes
tt0109831_shot_0298_img_0.jpg Is the person inside the red bounding box called Renée Zellweger? Please answer yes or no. No
tt0111280_shot_0258_img_0.jpg Is the actor inside the red bounding box named Gates McFadden? Please answer yes or no. Yes
tt0111280_shot_0258_img_0.jpg Is the actor inside the red bounding box named Michael Angarano? Please answer yes or no. No
tt0111280_shot_1479_img_2.jpg Is the actor inside the red bounding box called William Shatner? Please answer yes or no. Yes
tt0111280_shot_1479_img_2.jpg Is the actor inside the red bounding box called Richard Rohrbough? Please answer yes or no. No
tt0112384_shot_0878_img_0.jpg Is the person inside the red bounding box called Kathleen Quinlan? Please answer yes or no. Yes
tt0112384_shot_0878_img_0.jpg Is the person inside the red bounding box called Veronica Diaz Carranza? Please answer yes or no. No
tt0112641_shot_0412_img_1.jpg Is the actor inside the red bounding box called Robert De Niro? Please answer yes or no. Yes
tt0112641_shot_0412_img_1.jpg Is the actor inside the red bounding box called Pierre Malherbe? Please answer yes or no. No
tt0112740_shot_1056_img_0.jpg Is the person inside the red bounding box named Denzel Washington? Please answer yes or no. Yes
tt0112740_shot_1056_img_0.jpg Is the person inside the red bounding box named Bill Pullman? Please answer yes or no. No
tt0113101_shot_0547_img_0.jpg Is the person inside the red bounding box named Tim Roth? Please answer yes or no. Yes
tt0113101_shot_0547_img_0.jpg Is the person inside the red bounding box named Honey Chhaya? Please answer yes or no. No
tt0114369_shot_1138_img_0.jpg Is the person inside the red bounding box named Brad Pitt? Please answer yes or no. Yes
tt0114369_shot_1138_img_0.jpg Is the person inside the red bounding box named Benjamin Nitze? Please answer yes or no. No
tt0114388_shot_0162_img_0.jpg Is the actor inside the red bounding box called Emma Thompson? Please answer yes or no. Yes
tt0114388_shot_0162_img_0.jpg Is the actor inside the red bounding box called Francis P. Hughes? Please answer yes or no. No
tt0114388_shot_1207_img_1.jpg Is the person inside the red bounding box called Hugh Grant? Please answer yes or no. Yes
tt0114388_shot_1207_img_1.jpg Is the person inside the red bounding box called Zach Hopkins? Please answer yes or no. No
tt0115798_shot_0844_img_1.jpg Is the person inside the red bounding box named Jim Carrey? Please answer yes or no. Yes
tt0115798_shot_0844_img_1.jpg Is the person inside the red bounding box named Renee Herlocker? Please answer yes or no. No
tt0116367_shot_0755_img_0.jpg Is the actor inside the red bounding box named George Clooney? Please answer yes or no. Yes
tt0116367_shot_0755_img_0.jpg Is the actor inside the red bounding box named Ben Crowley? Please answer yes or no. No
tt0116629_shot_1570_img_2.jpg Is the person inside the red bounding box called Will Smith? Please answer yes or no. Yes
tt0116629_shot_1570_img_2.jpg Is the person inside the red bounding box called E. Katherine Kerr? Please answer yes or no. No
tt0116695_shot_0343_img_0.jpg Is the person inside the red bounding box named Tom Cruise? Please answer yes or no. Yes
tt0116695_shot_0343_img_0.jpg Is the person inside the red bounding box named Billy Dee? Please answer yes or no. No
tt0117060_shot_0412_img_0.jpg Is the actor inside the red bounding box called Tom Cruise? Please answer yes or no. Yes
tt0117060_shot_0412_img_0.jpg Is the actor inside the red bounding box called Carrie Lazar? Please answer yes or no. No
tt0117060_shot_1401_img_0.jpg Is the actor inside the red bounding box called Jean Reno? Please answer yes or no. Yes
tt0117060_shot_1401_img_0.jpg Is the actor inside the red bounding box called Jill Teed? Please answer yes or no. No
tt0117381_shot_0798_img_1.jpg Is the person inside the red bounding box called Edward Norton? Please answer yes or no. Yes
tt0117381_shot_0798_img_1.jpg Is the person inside the red bounding box called Michael Tezla? Please answer yes or no. No
tt0117500_shot_2467_img_0.jpg Is the actor inside the red bounding box called Ed Harris? Please answer yes or no. Yes
tt0117500_shot_2467_img_0.jpg Is the actor inside the red bounding box called Paul J.Q. Lee? Please answer yes or no. No
tt0117509_shot_0041_img_0.jpg Is the actor inside the red bounding box named Paul Rudd? Please answer yes or no. Yes
tt0117509_shot_0041_img_0.jpg Is the actor inside the red bounding box named Max Martini? Please answer yes or no. No
tt0117571_shot_0475_img_0.jpg Is the person inside the red bounding box named Neve Campbell? Please answer yes or no. Yes
tt0117571_shot_0475_img_0.jpg Is the person inside the red bounding box named Frank Hoyt Taylor? Please answer yes or no. No
tt0117731_shot_0300_img_0.jpg Is the actor inside the red bounding box called Patrick Stewart? Please answer yes or no. Yes
tt0117731_shot_0300_img_0.jpg Is the actor inside the red bounding box called Debra Montague? Please answer yes or no. No
tt0117731_shot_1067_img_0.jpg Is the actor inside the red bounding box called Patrick Stewart? Please answer yes or no. Yes
tt0117731_shot_1067_img_0.jpg Is the actor inside the red bounding box called Jenny Wilson? Please answer yes or no. No
tt0118548_shot_1296_img_0.jpg Is the actor inside the red bounding box called Clint Eastwood? Please answer yes or no. Yes
tt0118548_shot_1296_img_0.jpg Is the actor inside the red bounding box called Kate Winslet? Please answer yes or no. No
tt0118571_shot_0627_img_0.jpg Is the actor inside the red bounding box called Glenn Close? Please answer yes or no. Yes
tt0118571_shot_0627_img_0.jpg Is the actor inside the red bounding box called Arlene Farber? Please answer yes or no. No
tt0118636_shot_0007_img_1.jpg Is the person inside the red bounding box called Brad Renfro? Please answer yes or no. Yes
tt0118636_shot_0007_img_1.jpg Is the person inside the red bounding box called Sandra Park? Please answer yes or no. No
tt0118636_shot_0344_img_0.jpg Is the actor inside the red bounding box called Brad Renfro? Please answer yes or no. Yes
tt0118636_shot_0344_img_0.jpg Is the actor inside the red bounding box called Karen Strassman? Please answer yes or no. No
tt0118655_shot_0279_img_0.jpg Is the person inside the red bounding box called Robert Wagner? Please answer yes or no. Yes
tt0118655_shot_0279_img_0.jpg Is the person inside the red bounding box called Arthur Birnbaum? Please answer yes or no. No
tt0118655_shot_1152_img_2.jpg Is the actor inside the red bounding box called Seth Green? Please answer yes or no. Yes
tt0118655_shot_1152_img_2.jpg Is the actor inside the red bounding box called Sue Doucette? Please answer yes or no. No
tt0118689_shot_0706_img_0.jpg Is the actor inside the red bounding box called Rowan Atkinson? Please answer yes or no. Yes
tt0118689_shot_0706_img_0.jpg Is the actor inside the red bounding box called Hugo Perez? Please answer yes or no. No
tt0118689_shot_0969_img_2.jpg Is the actor inside the red bounding box called Rowan Atkinson? Please answer yes or no. Yes
tt0118689_shot_0969_img_2.jpg Is the actor inside the red bounding box called Jack Shields? Please answer yes or no. No
tt0118715_shot_0079_img_0.jpg Is the actor inside the red bounding box called Jeff Bridges? Please answer yes or no. Yes
tt0118715_shot_0079_img_0.jpg Is the actor inside the red bounding box called Scott Adkins? Please answer yes or no. No
tt0118749_shot_0795_img_0.jpg Is the person inside the red bounding box called John C. Reilly? Please answer yes or no. Yes
tt0118749_shot_0795_img_0.jpg Is the person inside the red bounding box called Chris Lowell? Please answer yes or no. No
tt0118883_shot_0691_img_1.jpg Is the actor inside the red bounding box called Julia Roberts? Please answer yes or no. Yes
tt0118883_shot_0691_img_1.jpg Is the actor inside the red bounding box called Roger Bart? Please answer yes or no. No
tt0118971_shot_0679_img_0.jpg Is the actor inside the red bounding box called Charlize Theron? Please answer yes or no. Yes
tt0118971_shot_0679_img_0.jpg Is the actor inside the red bounding box called Young-min Kim? Please answer yes or no. No
tt0119008_shot_0979_img_0.jpg Is the actor inside the red bounding box named Al Pacino? Please answer yes or no. Yes
tt0119008_shot_0979_img_0.jpg Is the actor inside the red bounding box named Neil Tweddle? Please answer yes or no. No
tt0119094_shot_0446_img_2.jpg Is the actor inside the red bounding box called Nicolas Cage? Please answer yes or no. Yes
tt0119094_shot_0446_img_2.jpg Is the actor inside the red bounding box called Juan Gabriel Pareja? Please answer yes or no. No
tt0119116_shot_0721_img_0.jpg Is the actor inside the red bounding box called Bruce Willis? Please answer yes or no. Yes
tt0119116_shot_0721_img_0.jpg Is the actor inside the red bounding box called Troye Sivan? Please answer yes or no. No
tt0119174_shot_0439_img_0.jpg Is the actor inside the red bounding box named Michael Douglas? Please answer yes or no. Yes
tt0119174_shot_0439_img_0.jpg Is the actor inside the red bounding box named Carola McGuinness? Please answer yes or no. No
tt0119314_shot_0572_img_0.jpg Is the actor inside the red bounding box called Scarlett Johansson? Please answer yes or no. Yes
tt0119314_shot_0572_img_0.jpg Is the actor inside the red bounding box called Daisy Beaumont? Please answer yes or no. No
tt0119528_shot_0171_img_0.jpg Is the person inside the red bounding box called Jim Carrey? Please answer yes or no. Yes
tt0119528_shot_0171_img_0.jpg Is the person inside the red bounding box called Eliot Paton? Please answer yes or no. No
tt0119528_shot_0761_img_1.jpg Is the actor inside the red bounding box named Jim Carrey? Please answer yes or no. Yes
tt0119528_shot_0761_img_1.jpg Is the actor inside the red bounding box named Jari Kinnunen? Please answer yes or no. No
tt0119643_shot_0330_img_0.jpg Is the actor inside the red bounding box named Brad Pitt? Please answer yes or no. Yes
tt0119643_shot_0330_img_0.jpg Is the actor inside the red bounding box named Anthony Hopkins? Please answer yes or no. No
tt0119738_shot_0201_img_0.jpg Is the person inside the red bounding box named Christopher Masterson? Please answer yes or no. Yes
tt0119738_shot_0201_img_0.jpg Is the person inside the red bounding box named Edwin Craig? Please answer yes or no. No
tt0119822_shot_0878_img_0.jpg Is the person inside the red bounding box named Greg Kinnear? Please answer yes or no. Yes
tt0119822_shot_0878_img_0.jpg Is the person inside the red bounding box named Aleksandr Dubina? Please answer yes or no. No
tt0120338_shot_0444_img_2.jpg Is the actor inside the red bounding box named Kate Winslet? Please answer yes or no. Yes
tt0120338_shot_0444_img_2.jpg Is the actor inside the red bounding box named Donald Gibb? Please answer yes or no. No
tt0120338_shot_1130_img_2.jpg Is the person inside the red bounding box called Leonardo DiCaprio? Please answer yes or no. Yes
tt0120338_shot_1130_img_2.jpg Is the person inside the red bounding box called Anne Betancourt? Please answer yes or no. No
internvl_chat/eval/mme/Your_Results/code_reasoning.txt 0 → 100644
View file @ 26e59280
0001.png The image shows a python code. Is the output of the code 'Hello'? Please answer yes or no. Yes
0001.png The image shows a python code. Is the output of the code 'World'? Please answer yes or no. No
0002.png The image shows a python code. Is the output of the code 'a cat'? Please answer yes or no. Yes
0002.png The image shows a python code. Is the output of the code 'a dog'? Please answer yes or no. No
0003.png The image shows a python code. Is the output of the code '12'? Please answer yes or no. Yes
0003.png The image shows a python code. Is the output of the code '5'? Please answer yes or no. No
0004.png The image shows a python code. Is the output of the code '3'? Please answer yes or no. Yes
0004.png The image shows a python code. Is the output of the code '2'? Please answer yes or no. No
0005.png The image shows a python code. Is the output of the code '12'? Please answer yes or no. Yes
0005.png The image shows a python code. Is the output of the code '5'? Please answer yes or no. No
0006.png The image shows a python code. Is the output of the code '0'? Please answer yes or no. Yes
0006.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. No
0007.png Is a c++ code shown in the picture? Please answer yes or no. Yes
0007.png Is a python code shown in the picture? Please answer yes or no. No
0008.png The image shows a python code. Is the output of the code '1234'? Please answer yes or no. Yes
0008.png The image shows a python code. Is the output of the code '12345'? Please answer yes or no. No
0009.png The image shows a python code. Is the output of the code '36'? Please answer yes or no. Yes
0009.png The image shows a python code. Is the output of the code '6'? Please answer yes or no. No
0010.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. Yes
0010.png The image shows a python code. Is the output of the code '5'? Please answer yes or no. No
0011.png The image shows a python code. Is the output of the code '0'? Please answer yes or no. Yes
0011.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. No
0012.png The image shows a python code. Is the output of the code 'working hard'? Please answer yes or no. Yes
0012.png The image shows a python code. Is the output of the code 'playing hard'? Please answer yes or no. No
0013.png The image shows a python code. Is the output of the code 'a cat'? Please answer yes or no. Yes
0013.png The image shows a python code. Is the output of the code 'a dog'? Please answer yes or no. No
0014.png The image shows a python code. Is the output of the code '7'? Please answer yes or no. Yes
0014.png The image shows a python code. Is the output of the code '1'? Please answer yes or no. No
0015.png The image shows a python code. Is the output of the code '11'? Please answer yes or no. Yes
0015.png The image shows a python code. Is the output of the code '9'? Please answer yes or no. No
0016.png The image shows a python code. Is the output of the code 'x is smaller than 10'? Please answer yes or no. Yes
0016.png The image shows a python code. Is the output of the code 'x is larger than 10'? Please answer yes or no. No
0017.png The image shows a python code. Will the number 3 appear in the output of the code? Please answer yes or no. Yes
0017.png The image shows a python code. Will the number 6 appear in the output of the code? Please answer yes or no. No
0018.png The image shows a python code. Is the output of the code '11'? Please answer yes or no. Yes
0018.png The image shows a python code. Is the output of the code '12'? Please answer yes or no. No
0019.png The image shows a python code. Is the output of the code 'the list has more than 2 numbers'? Please answer yes or no. Yes
0019.png The image shows a python code. Is the output of the code 'the list has less than 2 numbers'? Please answer yes or no. No
0020.png Is a python code shown in the picture? Please answer yes or no. Yes
0020.png Is a c++ code shown in the picture? Please answer yes or no. No
internvl_chat/eval/mme/Your_Results/color.txt 0 → 100644
View file @ 26e59280
000000006723.jpg Is there a red brick building in the image? Please answer yes or no. Yes
000000006723.jpg Is there a yellow brick building in the image? Please answer yes or no. No
000000008277.jpg Is there a white plate in the image? Please answer yes or no. Yes
000000008277.jpg Is there a yellow plate in the image? Please answer yes or no. No
000000012120.jpg Is there a blue court in the image? Please answer yes or no. Yes
000000012120.jpg Is there a purple court in the image? Please answer yes or no. No
000000014831.jpg Is there a brown and white animal in the image? Please answer yes or no. Yes
000000014831.jpg Is there a green and red animal in the image? Please answer yes or no. No
000000028993.jpg Are there yellow poles in the image? Please answer yes or no. Yes
000000028993.jpg Are there blue poles in the image? Please answer yes or no. No
000000029393.jpg Is there a brown dog in the image? Please answer yes or no. Yes
000000029393.jpg Is there a black dog in the image? Please answer yes or no. No
000000035770.jpg Is there a black and white toilet in the image? Please answer yes or no. Yes
000000035770.jpg Is there a red and white toilet in the image? Please answer yes or no. No
000000038118.jpg Is there a red coat in the image? Please answer yes or no. Yes
000000038118.jpg Is there a yellow coat in the image? Please answer yes or no. No
000000047112.jpg Is there a white plate in the image? Please answer yes or no. Yes
000000047112.jpg Is there a yellow plate in the image? Please answer yes or no. No
000000047121.jpg Is there a black cat in the image? Please answer yes or no. Yes
000000047121.jpg Is there a brown cat in the image? Please answer yes or no. No
000000053529.jpg Is there a green hat in the image? Please answer yes or no. Yes
000000053529.jpg Is there a red hat in the image? Please answer yes or no. No
000000053994.jpg Is there a gray wall in the image? Please answer yes or no. Yes
000000053994.jpg Is there a red wall in the image? Please answer yes or no. No
000000055072.jpg Is there a brown giraffe in the image? Please answer yes or no. Yes
000000055072.jpg Is there a black giraffe in the image? Please answer yes or no. No
000000057597.jpg Are there any red shoes in the image? Please answer yes or no. Yes
000000057597.jpg Are there any yellow shoes in the image? Please answer yes or no. No
000000061658.jpg Are there a white dish in the image? Please answer yes or no. Yes
000000061658.jpg Are there a green dish in the image? Please answer yes or no. No
000000338560.jpg Is there a blue and yellow fire hydrant in the image? Please answer yes or no. Yes
000000338560.jpg Is there a blue and orange fire hydrant in the image? Please answer yes or no. No
000000370208.jpg Is there a red bicycle with white handlebars in the image? Please answer yes or no. Yes
000000370208.jpg Is there a red bicycle with black handlebars in the image? Please answer yes or no. No
000000377723.jpg Is there a blue bus in the image? Please answer yes or no. Yes
000000377723.jpg Is there a orange bus in the image? Please answer yes or no. No
000000405205.jpg Is there a white bus in the image? Please answer yes or no. Yes
000000405205.jpg Is there a red bus in the image? Please answer yes or no. No
000000410612.jpg Is there a red boat in the image? Please answer yes or no. Yes
000000410612.jpg Is there a gray boat in the image? Please answer yes or no. No
000000427034.jpg Is there a brown and black dog in the image? Please answer yes or no. Yes
000000427034.jpg Is there a brown and white dog in the image? Please answer yes or no. No
000000442456.jpg Is there a man wearing a red shirt in the image? Please answer yes or no. Yes
000000442456.jpg Is there a man wearing a white shirt in the image? Please answer yes or no. No
000000492362.jpg Is there a skateboard with red wheels in the image? Please answer yes or no. Yes
000000492362.jpg Is there a skateboard with black wheels in the image? Please answer yes or no. No
000000492992.jpg Is there a white bird in the image? Please answer yes or no. Yes
000000492992.jpg Is there a yellow bird in the image? Please answer yes or no. No
000000512929.jpg Are there any green beans in the image? Please answer yes or no. Yes
000000512929.jpg Are there any orange beans in the image? Please answer yes or no. No
000000530457.jpg Are there any red flowers in the image? Please answer yes or no. Yes
000000530457.jpg Are there any green flowers in the image? Please answer yes or no. No
000000532761.jpg Is there a living room painted yellow in the image? Please answer yes or no. Yes
000000532761.jpg Is there a living room painted black in the image? Please answer yes or no. No
000000534041.jpg Is there a purple bottle in the image? Please answer yes or no. Yes
000000534041.jpg Is there a white bottle in the image? Please answer yes or no. No
000000563758.jpg Is there a red scarf in the image? Please answer yes or no. Yes
000000563758.jpg Is there a brown scarf in the image? Please answer yes or no. No
000000564280.jpg Is there a red couch in the image? Please answer yes or no. Yes
000000564280.jpg Is there a black couch in the image? Please answer yes or no. No
internvl_chat/eval/mme/Your_Results/commonsense_reasoning.txt 0 → 100644
View file @ 26e59280
0001.png Here are the order details for my taxi ride. Should I actually pay $29.42? Please answer yes or no. Yes
0001.png Here are the order details for my taxi ride. Should I actually pay $32.42? Please answer yes or no. No
0002.png Should I stop when I'm about to cross the street and see the sign in the picture? Please answer yes or no. Yes
0002.png When I see the sign in the picture, can I cross the street? Please answer yes or no. No
0003.png May I ask if in the game of finger-guessing game, did the right side of the picture win? Please answer yes or no. Yes
0003.png May I ask if in the game of finger-guessing game, did the left side of the picture win? Please answer yes or no. No
0004.png Does the fruit in the picture look stale? Please answer yes or no. Yes
0004.png Does the fruit in the picture look very fresh? Please answer yes or no. No
0005.png The office's normal closing time is 5 p.m. Now is afternoon. Should I continue to work at the time shown in the picture? Please answer yes or no. Yes
0005.png The office's normal closing time is 5 p.m. Now is afternoon. Could I leave work at the time shown in the picture? Please answer yes or no. No
0006.png I recently want to go on vacation to relax and go to a place full of fresh air. Is the venue in the picture appropriate? Please answer yes or no. Yes
0006.png I want to go where there are a lot of people. Is the venue in the picture appropriate? Please answer yes or no. No
0007.png I want to clean the house and I want to choose a tool. Is the tool in the picture an appropriate choice? Please answer yes or no. Yes
0007.png I want to transport something and I want to choose a tool to help me. Is the tool in the picture an appropriate choice? Please answer yes or no. No
0008.png Can I smoke where the picture is? Please answer yes or no. Yes
0008.png Is smoking prohibited in the location of the picture? Please answer yes or no. No
0009.png Will green be obtained by mixing the above two colors? Please answer yes or no. Yes
0009.png Will red be obtained by mixing the above two colors? Please answer yes or no. No
0010.png I am going to exercise and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. Yes
0010.png I am going to study and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. No
0011.png If I am allergic to durian, can I finish the fruit in the picture? Please answer yes or no. Yes
0011.png If I am allergic to banana, can I finish the fruit in the picture? Please answer yes or no. No
0012.png I am going to study and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. Yes
0012.png I am going to exercise and I want to choose a venue. Is the venue in the picture a good choice? Please answer yes or no. No
0013.png I am going to a formal dinner party. Is the shoe in the picture an appropriate choice? Please answer yes or no. Yes
0013.png I am going to play basketball. Is the shoe in the picture an appropriate choice? Please answer yes or no. No
0014.png In this line chart, the vertical axis is height and the horizontal axis is age. Does Maria's height exceed Jane's height in the end? Please answer yes or no. Yes
0014.png In this line chart, the vertical axis is height and the horizontal axis is age. Does Jane's height exceed Kangkang's height in the end? Please answer yes or no. No
0015.png Is the ball usually played with hands? Please answer yes or no. Yes
0015.png Is the ball usually played with feet? Please answer yes or no. No
0016.png Is the place in the picture a good place to enjoy the cool in a sunny day? Please answer yes or no. Yes
0016.png Is the place in the picture a good shelter from the rain when it thunders outside? Please answer yes or no. No
0017.png Are the vehicles in the pictures usually environmentally friendly? Please answer yes or no. Yes
0017.png Does the vehicle in the picture usually run faster than the car? Please answer yes or no. No
0018.png This is a picture of some kind of animal. Does it eat leaves? Please answer yes or no. Yes
0018.png This is a picture of some kind of animal. Does it eat meat? Please answer yes or no. No
0019.png Is the water flow in the picture from the top to the bottom? Please answer yes or no. Yes
0019.png Is the water flow in the picture from the bottom to the top? Please answer yes or no. No
0020.png Can the item in the picture be used to measure length? Please answer yes or no. Yes
0020.png Can the item in the picture be used to measure angles? Please answer yes or no. No
0021.png This is a toilet guide sign. I am a man. Should I go to the toilet on the left? Please answer yes or no. Yes
0021.png This is a toilet guide sign. I am a man. Should I go to the toilet on the right? Please answer yes or no. No
0022.png Does the animal in the picture usually catch mice? Please answer yes or no. Yes
0022.png Is the animal in the picture usually used in search and rescue? Please answer yes or no. No
0023.png If you want to keep your fruit fresh in summer, should you put it in the appliance in the picture? Please answer yes or no. Yes
0023.png Is the appliance in the picture more suitable for winter than summer? Please answer yes or no. No
0024.png I want to go skating. Is the shoe in the picture usually appropriate? Please answer yes or no. Yes
0024.png I want to go roller skating. Is the shoe in the picture usually appropriate? Please answer yes or no. No
0025.png I feel very thirsty in the desert now. Can the thing in the picture help me? Please answer yes or no. Yes
0025.png I don't like clear cups. Is the cup in the picture my type? Please answer yes or no. No
0026.png I want to go for a run and I want to choose a pair of shoes. Is the shoe in the picture an appropriate choice? Please answer yes or no. Yes
0026.png I want to practice ballet and I want to choose a pair of shoes. Is the shoe in the picture an appropriate choice? Please answer yes or no. No
0027.png Are the pants in the picture usually suitable for casual wear? Please answer yes or no. Yes
0027.png Are the pants in the picture usually suitable for playing basketball? Please answer yes or no. No
0028.png This is a picture from a real scene. Is there only one real cat in this picture? Please answer yes or no. Yes
0028.png This is a picture from a real scene. Is there only two real cats in this picture? Please answer yes or no. No
0029.png The three cats in the picture, the one without a beard, is the middle one? Please answer yes or no. Yes
0029.png The three cats in the picture, the one without a beard, is the right one? Please answer yes or no. No
0030.png I'm going to 501. Do I need to turn left at the intersection? Please answer yes or no. Yes
0030.png I'm going to 502. Do I need to turn left at the intersection? Please answer yes or no. No
0031.png Is the drink in the picture usually suitable for a party? Please answer yes or no. Yes
0031.png Is the drink in the picture usually suitable for drinking together with cephalosporin? Please answer yes or no. No
0032.png Here is a picture of the cake I cut. Did I cut it at least twice? Please answer yes or no. Yes
0032.png Here is a picture of the cake I cut. Did I cut it at least once? Please answer yes or no. No
0033.png This is a picture from a real scene. Is there only one real apple in this picture? Please answer yes or no. Yes
0033.png This is a picture from a real scene. Is there only two real apples in this picture? Please answer yes or no. No
0034.png Here is a pie chart counting the favorite fruits of all employees in our company. Is the durian the most popular fruit? Please answer yes or no. Yes
0034.png Here is a pie chart counting the favorite fruits of all employees in our company. Is the mango the most popular fruit? Please answer yes or no. No
0035.png This is the sales chart of this month. Is Tina the runner-up in sales this month? Please answer yes or no. Yes
0035.png This is the sales chart of this month. Is John the runner-up in sales this month? Please answer yes or no. No
0036.png Is it a good time to walk through the road in the picture? Please answer yes or no. Yes
0036.png Is it a good time to drive a car through the road in the picture? Please answer yes or no. No
0037.png Here is a photo of the sun's position at a certain time. Could it be dusk now? Please answer yes or no. Yes
0037.png Here is a photo of the sun's position at a certain time. Could it be noon now? Please answer yes or no. No
0038.png All apples are shown in the picture. If I eat an apple every day, can I eat it for four days? Please answer yes or no. Yes
0038.png All apples are shown in the picture. If I eat an apple every day, can I eat it for three days? Please answer yes or no. No
0039.png This line chart is used to count the sales of two types of burgers. Are chicken burgers more popular? Please answer yes or no. Yes
0039.png This line chart is used to count the sales of two types of burgers. Are beef burgers more popular? Please answer yes or no. No
0040.png I want to supplement protein. Is it appropriate to eat the food in the picture? Please answer yes or no. Yes
0040.png I don't like to eat any food related to chicken. Is the food in the picture my type? Please answer yes or no. No
0041.png Is the fruit in the picture usually sweet? Please answer yes or no. Yes
0041.png Is the fruit in the picture usually spicy? Please answer yes or no. No
0042.png Are there usually cars in the area shown in the picture? Please answer yes or no. Yes
0042.png Is it appropriate to cross the road directly from the place shown in the picture? Please answer yes or no. No
0043.png Is the animal in the picture usually not seen in winter? Please answer yes or no. Yes
0043.png Is the animal in the picture usually seen in winter? Please answer yes or no. No
0044.png This is a flowchart of a program. I enter 3 and 6. Is the output 'No'? Please answer yes or no. Yes
0044.png This is a flowchart of a program. I enter 3 and 6. Is the output 'Yes'? Please answer yes or no. No
0045.png There is a sign at the intersection, can I turn left? Please answer yes or no. Yes
0045.png There is a sign at the intersection, can I turn right? Please answer yes or no. No
0046.png Vitamin C is very helpful for human health. Does the food on in the picture usually contain Vitamin C? Please answer yes or no. Yes
0046.png Is the food in the picture commonly used to build muscle? Please answer yes or no. No
0047.png All apples are shown in the picture. My brother and I divide the apples equally. May I have one apple? Please answer yes or no. Yes
0047.png All apples are shown in the picture. My brother and I divide the apples equally. May I have two apples? Please answer yes or no. No
0048.png Here is a picture of eating fruit. Am I eating a strawberry? Please answer yes or no. Yes
0048.png Here is a picture of eating fruit. Am I eating a cherry tomato? Please answer yes or no. No
0049.png Does the vehicle in the picture usually have its Windows closed during fast driving? Please answer yes or no. Yes
0049.png Does the vehicle in the picture usually have its Windows opened during fast driving? Please answer yes or no. No
0050.png Do people commonly use the item in the picture for makeup in their daily lives? Please answer yes or no. Yes
0050.png Do people commonly use the item in the picture to write in their daily lives? Please answer yes or no. No
0051.png This is a flowchart of a program. When the input is 5, is the output 6? Please answer yes or no. Yes
0051.png This is a flowchart of a program. When the input is 6, is the output 5? Please answer yes or no. No
0052.png I want to lose weight. Is the food in the picture an appropriate choice? Please answer yes or no. Yes
0052.png I want to gain weight. Is the food in the picture an appropriate choice? Please answer yes or no. No
0053.png Is the car in the picture going to make a right turn after going through a straight road section? Please answer yes or no. Yes
0053.png Is the car in the picture going to make a left turn after going through a straight road section? Please answer yes or no. No
0054.png May I ask if the plants in the picture can survive in the water? Please answer yes or no. Yes
0054.png May I ask if the plants in the picture can survive in the soil? Please answer yes or no. No
0055.png The man in the picture is eating. Does he eat noodles? Please answer yes or no. Yes
0055.png The man in the picture is eating. Does he eat rice? Please answer yes or no. No
0056.png Can the item in the picture output water? Please answer yes or no. Yes
0056.png Can the item in picture be used for blowing air? Please answer yes or no. No
0057.png Does the vehicle in the picture usually run faster than a horse? Please answer yes or no. Yes
0057.png Does the vehicle in the picture usually fly? Please answer yes or no. No
0058.png Can't I smoke here? Please answer yes or no. Yes
0058.png May I smoke here? Please answer yes or no. No
0059.png This pie chart is the age distribution of our company. Is the proportion of people aged 30-50 more than 40%? Please answer yes or no. Yes
0059.png This pie chart is the age distribution of our company. Is the proportion of people aged 40-50 more than 30%? Please answer yes or no. No
0060.png This is the histogram of fruit sales today. Do more men buy watermelons than women buy bananas? Please answer yes or no. Yes
0060.png This is the histogram of fruit sales today. Do more men buy peach than women buy apple? Please answer yes or no. No
0061.png Is the tool in the picture common in tall buildings? Please answer yes or no. Yes
0061.png In case of fire, is it appropriate to choose the tool in the picture to go downstairs? Please answer yes or no. No
0062.png It's snowing outside the window now. I want to go out. Is it appropriate to wear the cloth in the picture? Please answer yes or no. Yes
0062.png It's very hot outside. I want to go out. Is it appropriate to wear the cloth in the picture? Please answer yes or no. No
0063.png Is the animal in the picture suitable as a pet? Please answer yes or no. Yes
0063.png Is the animal in the pictures usually stronger than adult tigers? Please answer yes or no. No
0064.png I want to play basketball. Is the venue in the picture a good choice? Please answer yes or no. Yes
0064.png I want to play football. Is the venue in the picture a good choice? Please answer yes or no. No
0065.png Is it appropriate to wear a down jacket during the season in the picture? Please answer yes or no. Yes
0065.png Is it appropriate to only wear short sleeves during the season in the picture? Please answer yes or no. No
0066.png I want to carry one thing with me on a rainy day. Is the thing in the image an appropriate choice? Please answer yes or no. Yes
0066.png It is raining outside. I am in a house and I don't need to go out. Is this thing in the picture necessary for me to use? Please answer yes or no. No
0067.png I feel very hot. Is the tool in the picture suitable for use? Please answer yes or no. Yes
0067.png I feel very cold. Is the tool in the picture suitable for use? Please answer yes or no. No
0068.png Is it unhealthy to eat the food in the picture too often? Please answer yes or no. Yes
0068.png Is the food in the picture usually low in calories? Please answer yes or no. No
0069.png Is the phone in the photo connected to a charger? Please answer yes or no. Yes
0069.png Is the phone in the photo charging? Please answer yes or no. No
0070.png I want to turn the screw. Is the tool in the picture usually appropriate? Please answer yes or no. Yes
0070.png Is the tool in the picture usually suitable for smashing walnuts? Please answer yes or no. No
internvl_chat/eval/mme/Your_Results/count.txt 0 → 100644
View file @ 26e59280
000000006040.jpg Is there a train in the picture? Please answer yes or no. Yes
000000006040.jpg Are there a total of two trains in the picture? Please answer yes or no. No
000000044279.jpg Is there a total of two people in the image? Please answer yes or no. Yes
000000044279.jpg Is there only one people in the image? Please answer yes or no. No
000000067213.jpg Is there only one dog in the image? Please answer yes or no. Yes
000000067213.jpg Is there two dogs in the image? Please answer yes or no. No
000000071226.jpg Is there a total of two dogs in the image? Please answer yes or no. Yes
000000071226.jpg Is there only one dogs in the image? Please answer yes or no. No
000000097994.jpg Are there three laptops in the picture? Please answer yes or no. Yes
000000097994.jpg Are there four laptops in the picture? Please answer yes or no. No
000000195918.jpg Is there a total of two display devices in the image? Please answer yes or no. Yes
000000195918.jpg Is there only one display device in the image? Please answer yes or no. No
000000236721.jpg Are there two bananas in the image? Please answer yes or no. Yes
000000236721.jpg Are there three bananas in the image? Please answer yes or no. No
000000261712.jpg Are there two giraffes in this image? Please answer yes or no. Yes
000000261712.jpg Are there three giraffes in this picture? Please answer yes or no. No
000000274066.jpg Are there four people appear in this image? Please answer yes or no. Yes
000000274066.jpg Are there only three people appear in this image? Please answer yes or no. No
000000276434.jpg Is there a total of three cakes in this image? Please answer yes or no. Yes
000000276434.jpg Are there only two cakes in this image? Please answer yes or no. No
000000289059.jpg Is there a total of two person appear in the image? Please answer yes or no. Yes
000000289059.jpg Is there only one person appear in the image? Please answer yes or no. No
000000290081.jpg Is there only one bowl in this image? Please answer yes or no. Yes
000000290081.jpg Are there two bowls in this image? Please answer yes or no. No
000000301867.jpg Are there three people appear in this image? Please answer yes or no. Yes
000000301867.jpg Are there only two people appear in this image? Please answer yes or no. No
000000335954.jpg Are there two bowls in this image? Please answer yes or no. Yes
000000335954.jpg Are there three bowls in this image? Please answer yes or no. No
000000357816.jpg Are there four people in this image? Please answer yes or no. Yes
000000357816.jpg Are there five people in this image? Please answer yes or no. No
000000372819.jpg Are there four dogs appear in this image? Please answer yes or no. Yes
000000372819.jpg Are there only three dogs appear in this image? Please answer yes or no. No
000000410612.jpg Is there only one ship in the picture? Please answer yes or no. Yes
000000410612.jpg Is there a total of two ships in the picture? Please answer yes or no. No
000000423944.jpg Is there no person in this picture? Please answer yes or no. Yes
000000423944.jpg Are there two people appear in this image? Please answer yes or no. No
000000427034.jpg Is there a dog in the picture? Please answer yes or no. Yes
000000427034.jpg Are there a total of two dogs in the picture? Please answer yes or no. No
000000430286.jpg Are there three remotes in this image? Please answer yes or no. Yes
000000430286.jpg Are there only two remotes in this image? Please answer yes or no. No
000000432468.jpg Are there three zippers in the picture? Please answer yes or no. Yes
000000432468.jpg Is there a zipper in the picture? Please answer yes or no. No
000000434479.jpg Are there two pieces of pizza in this image? Please answer yes or no. Yes
000000434479.jpg Is there only one piece of pizza in this image? Please answer yes or no. No
000000438304.jpg Are there two tennis rackets in the picture? Please answer yes or no. Yes
000000438304.jpg Are there only one tennis racket in the picture? Please answer yes or no. No
000000450303.jpg Are there six people appear in this image? Please answer yes or no. Yes
000000450303.jpg Are there seven people appear in this image? Please answer yes or no. No
000000470121.jpg Is there only one bottle in the image? Please answer yes or no. Yes
000000470121.jpg Is there two bottles in the image? Please answer yes or no. No
000000476215.jpg Are there two horses in this image? Please answer yes or no. Yes
000000476215.jpg Is there only one horse in this image? Please answer yes or no. No
000000482100.jpg Are there two toilets in the picture? Please answer yes or no. Yes
000000482100.jpg Is there only one toilet in the picture? Please answer yes or no. No
000000491867.jpg Is there only one necktie in the image? Please answer yes or no. Yes
000000491867.jpg Is there three neckties in the image? Please answer yes or no. No
000000556000.jpg Are there four people in the image? Please answer yes or no. Yes
000000556000.jpg Are there only three people in the image? Please answer yes or no. No
000000565045.jpg Are there two bath towels in the picture? Please answer yes or no. Yes
000000565045.jpg Is there only one bath towel in the picture? Please answer yes or no. No
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment