Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
xgen_mm.py 2.84 KB
Newer Older
luopl's avatar
Initial commit  
luopl committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81 
from PIL import Image
import torch

from .base import BaseModel
from ..smp import *


class XGenMM(BaseModel):

    INSTALL_REQ = False
    INTERLEAVE = True

    def __init__(self, model_path='Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5', **kwargs):
        try:
            from transformers import AutoModelForVision2Seq, AutoTokenizer, AutoImageProcessor
        except Exception as err:
            logging.critical('Please install the latest version transformers.')
            raise err

        model = AutoModelForVision2Seq.from_pretrained(
            model_path, device_map='cuda', trust_remote_code=True, torch_dtype='auto'
        ).eval()

        tokenizer = AutoTokenizer.from_pretrained(
            model_path, trust_remote_code=True, use_fast=False, legacy=False
        )
        tokenizer = model.update_special_tokens(tokenizer)
        tokenizer.eos_token = '<|end|>'
        tokenizer.padding_side = 'left'
        image_processor = AutoImageProcessor.from_pretrained(model_path, trust_remote_code=True)
        self.model = model
        self.image_processor = image_processor
        self.tokenizer = tokenizer
        self.kwargs = kwargs

    def apply_prompt_template(self, query):
        s = (
            '<|system|>\nA chat between a curious user and an artificial intelligence assistant. '
            "The assistant gives helpful, detailed, and polite answers to the user's questions.<|end|>\n"
            f'<|user|>\n{query}<|end|>\n<|assistant|>\n'
        )
        return s

    def generate_inner(self, message, dataset=None):

        content, images, image_sizes = '', [], []

        for msg in message:
            if msg['type'] == 'text':
                content += msg['value']
            elif msg['type'] == 'image':
                image = Image.open(msg['value']).convert('RGB')
                images.append(self.image_processor([image], image_aspect_ratio='anyres')['pixel_values'].to('cuda'))
                image_sizes.append(image.size)
                content += '<image> '

        inputs = {'pixel_values': [images]}
        prompt = self.apply_prompt_template(content)
        language_inputs = self.tokenizer([prompt], return_tensors='pt').to('cuda')
        inputs.update(language_inputs)

        generation_args = {
            'max_new_tokens': 1024,
            'temperature': 0.0,
            'do_sample': False,
            'top_p': None,
            'num_beams': 1
        }
        generation_args.update(self.kwargs)

        generate_ids = self.model.generate(
            **inputs, image_size=[image_sizes],
            pad_token_id=self.tokenizer.pad_token_id,
            eos_token_id=self.tokenizer.eos_token_id,
            **generation_args
        )

        # remove input tokens
        response = self.tokenizer.decode(generate_ids[0], skip_special_tokens=True).split('<|end|>')[0]

        return response