template.py 76.2 KB
Newer Older
chenych's avatar
chenych committed
1
# Copyright 2025 the LlamaFactory team.
chenych's avatar
chenych committed
2
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

chenych's avatar
chenych committed
15
import re
chenych's avatar
chenych committed
16
from copy import deepcopy
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
17
from dataclasses import dataclass
chenych's avatar
chenych committed
18
from typing import TYPE_CHECKING, Optional, Union
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
19

luopl's avatar
luopl committed
20
21
from typing_extensions import override

luopl's avatar
luopl committed
22
from ..extras import logging
chenych's avatar
chenych committed
23
from .data_utils import Role
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
24
from .formatter import EmptyFormatter, FunctionFormatter, StringFormatter, ToolFormatter
luopl's avatar
luopl committed
25
from .mm_plugin import get_mm_plugin
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
26
27
28
29
30


if TYPE_CHECKING:
    from transformers import PreTrainedTokenizer

luopl's avatar
luopl committed
31
    from ..hparams import DataArguments
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
32
    from .formatter import SLOTS, Formatter
luopl's avatar
luopl committed
33
    from .mm_plugin import BasePlugin
luopl's avatar
luopl committed
34
    from .tool_utils import FunctionCall
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
35
36


luopl's avatar
luopl committed
37
logger = logging.get_logger(__name__)
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
38
39
40
41
42
43
44
45
46
47


@dataclass
class Template:
    format_user: "Formatter"
    format_assistant: "Formatter"
    format_system: "Formatter"
    format_function: "Formatter"
    format_observation: "Formatter"
    format_tools: "Formatter"
chenych's avatar
chenych committed
48
    format_prefix: "Formatter"
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
49
    default_system: str
chenych's avatar
chenych committed
50
51
    stop_words: list[str]
    thought_words: tuple[str, str]
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
52
53
    efficient_eos: bool
    replace_eos: bool
luopl's avatar
luopl committed
54
    replace_jinja_template: bool
chenych's avatar
chenych committed
55
    enable_thinking: Optional[bool]
luopl's avatar
luopl committed
56
    mm_plugin: "BasePlugin"
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
57
58
59
60

    def encode_oneturn(
        self,
        tokenizer: "PreTrainedTokenizer",
chenych's avatar
chenych committed
61
        messages: list[dict[str, str]],
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
62
63
        system: Optional[str] = None,
        tools: Optional[str] = None,
chenych's avatar
chenych committed
64
65
    ) -> tuple[list[int], list[int]]:
        r"""Return a single pair of token ids representing prompt and response respectively."""
chenych's avatar
chenych committed
66
        encoded_messages = self._encode(tokenizer, messages, system, tools)
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
67
        prompt_ids = []
chenych's avatar
chenych committed
68
69
70
        for encoded_ids in encoded_messages[:-1]:
            prompt_ids += encoded_ids

chenych's avatar
chenych committed
71
72
        response_ids = encoded_messages[-1]
        return prompt_ids, response_ids
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
73
74
75
76

    def encode_multiturn(
        self,
        tokenizer: "PreTrainedTokenizer",
chenych's avatar
chenych committed
77
        messages: list[dict[str, str]],
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
78
79
        system: Optional[str] = None,
        tools: Optional[str] = None,
chenych's avatar
chenych committed
80
81
    ) -> list[tuple[list[int], list[int]]]:
        r"""Return multiple pairs of token ids representing prompts and responses respectively."""
chenych's avatar
chenych committed
82
        encoded_messages = self._encode(tokenizer, messages, system, tools)
chenych's avatar
chenych committed
83
84
        return [(encoded_messages[i], encoded_messages[i + 1]) for i in range(0, len(encoded_messages), 2)]

chenych's avatar
chenych committed
85
86
    def extract_tool(self, content: str) -> Union[str, list["FunctionCall"]]:
        r"""Extract tool message."""
chenych's avatar
chenych committed
87
        return self.format_tools.extract(content)
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
88

chenych's avatar
chenych committed
89
90
    def get_stop_token_ids(self, tokenizer: "PreTrainedTokenizer") -> list[int]:
        r"""Return stop token ids."""
luopl's avatar
luopl committed
91
92
93
94
95
96
        stop_token_ids = {tokenizer.eos_token_id}
        for token in self.stop_words:
            stop_token_ids.add(tokenizer.convert_tokens_to_ids(token))

        return list(stop_token_ids)

chenych's avatar
chenych committed
97
98
99
100
101
102
103
104
105
106
107
108
109
    def add_thought(self, content: str = "") -> str:
        r"""Add empty thought to assistant message."""
        return f"{self.thought_words[0]}\n\n{self.thought_words[1]}\n\n" + content

    def remove_thought(self, content: str) -> str:
        r"""Remove thought from assistant message."""
        pattern = re.compile(f"{re.escape(self.thought_words[0])}(.*?){re.escape(self.thought_words[1])}", re.DOTALL)
        return re.sub(pattern, "", content).lstrip("\n")

    def get_thought_word_ids(self, tokenizer: "PreTrainedTokenizer") -> list[int]:
        r"""Get the token ids of thought words."""
        return tokenizer.encode(self.add_thought(), add_special_tokens=False)

chenych's avatar
chenych committed
110
111
    def _convert_elements_to_ids(self, tokenizer: "PreTrainedTokenizer", elements: "SLOTS") -> list[int]:
        r"""Convert elements to token ids."""
chenych's avatar
chenych committed
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        token_ids = []
        for elem in elements:
            if isinstance(elem, str):
                if len(elem) != 0:
                    token_ids += tokenizer.encode(elem, add_special_tokens=False)
            elif isinstance(elem, dict):
                token_ids += [tokenizer.convert_tokens_to_ids(elem.get("token"))]
            elif isinstance(elem, set):
                if "bos_token" in elem and tokenizer.bos_token_id is not None:
                    token_ids += [tokenizer.bos_token_id]
                elif "eos_token" in elem and tokenizer.eos_token_id is not None:
                    token_ids += [tokenizer.eos_token_id]
            else:
                raise ValueError(f"Input must be string, set[str] or dict[str, str], got {type(elem)}")

        return token_ids

Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
129
130
131
    def _encode(
        self,
        tokenizer: "PreTrainedTokenizer",
chenych's avatar
chenych committed
132
        messages: list[dict[str, str]],
chenych's avatar
chenych committed
133
134
        system: Optional[str],
        tools: Optional[str],
chenych's avatar
chenych committed
135
136
137
    ) -> list[list[int]]:
        r"""Encode formatted inputs to pairs of token ids.

chenych's avatar
chenych committed
138
        Turn 0: prefix + system + query        resp
chenych's avatar
chenych committed
139
        Turn t: query                          resp.
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
140
141
142
143
144
        """
        system = system or self.default_system
        encoded_messages = []
        for i, message in enumerate(messages):
            elements = []
chenych's avatar
chenych committed
145
146
147
148
149
150
151

            if i == 0:
                elements += self.format_prefix.apply()
                if system or tools:
                    tool_text = self.format_tools.apply(content=tools)[0] if tools else ""
                    elements += self.format_system.apply(content=(system + tool_text))

chenych's avatar
chenych committed
152
            if message["role"] == Role.USER:
chenych's avatar
chenych committed
153
                elements += self.format_user.apply(content=message["content"], idx=str(i // 2))
chenych's avatar
chenych committed
154
            elif message["role"] == Role.ASSISTANT:
chenych's avatar
chenych committed
155
                elements += self.format_assistant.apply(content=message["content"])
chenych's avatar
chenych committed
156
            elif message["role"] == Role.OBSERVATION:
chenych's avatar
chenych committed
157
                elements += self.format_observation.apply(content=message["content"])
chenych's avatar
chenych committed
158
            elif message["role"] == Role.FUNCTION:
chenych's avatar
chenych committed
159
                elements += self.format_function.apply(content=message["content"])
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
160
161
162
163
164
            else:
                raise NotImplementedError("Unexpected role: {}".format(message["role"]))

            encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements))

chenych's avatar
chenych committed
165
        return encoded_messages
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
166

chenych's avatar
chenych committed
167
168
    @staticmethod
    def _add_or_replace_eos_token(tokenizer: "PreTrainedTokenizer", eos_token: str) -> None:
chenych's avatar
chenych committed
169
        r"""Add or replace eos token to the tokenizer."""
chenych's avatar
chenych committed
170
171
172
        if tokenizer.eos_token == eos_token:
            return

chenych's avatar
chenych committed
173
174
        is_added = tokenizer.eos_token_id is None
        num_added_tokens = tokenizer.add_special_tokens({"eos_token": eos_token})
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
175

chenych's avatar
chenych committed
176
177
178
179
180
181
182
183
184
        if is_added:
            logger.info_rank0(f"Add eos token: {tokenizer.eos_token}.")
        else:
            logger.info_rank0(f"Replace eos token: {tokenizer.eos_token}.")

        if num_added_tokens > 0:
            logger.warning_rank0("New tokens have been added, make sure `resize_vocab` is True.")

    def fix_special_tokens(self, tokenizer: "PreTrainedTokenizer") -> None:
chenych's avatar
chenych committed
185
        r"""Add eos token and pad token to the tokenizer."""
chenych's avatar
chenych committed
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
        stop_words = self.stop_words
        if self.replace_eos:
            if not stop_words:
                raise ValueError("Stop words are required to replace the EOS token.")

            self._add_or_replace_eos_token(tokenizer, eos_token=stop_words[0])
            stop_words = stop_words[1:]

        if tokenizer.eos_token_id is None:
            self._add_or_replace_eos_token(tokenizer, eos_token="<|endoftext|>")

        if tokenizer.pad_token_id is None:
            tokenizer.pad_token = tokenizer.eos_token
            logger.info_rank0(f"Add pad token: {tokenizer.pad_token}")

        if stop_words:
            num_added_tokens = tokenizer.add_special_tokens(
                dict(additional_special_tokens=stop_words), replace_additional_special_tokens=False
            )
            logger.info_rank0("Add {} to stop words.".format(",".join(stop_words)))
            if num_added_tokens > 0:
                logger.warning_rank0("New tokens have been added, make sure `resize_vocab` is True.")

    @staticmethod
    def _jinja_escape(content: str) -> str:
chenych's avatar
chenych committed
211
        r"""Escape single quotes in content."""
chenych's avatar
chenych committed
212
213
214
215
        return content.replace("'", r"\'")

    @staticmethod
    def _convert_slots_to_jinja(slots: "SLOTS", tokenizer: "PreTrainedTokenizer", placeholder: str = "content") -> str:
chenych's avatar
chenych committed
216
        r"""Convert slots to jinja template."""
chenych's avatar
chenych committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
        slot_items = []
        for slot in slots:
            if isinstance(slot, str):
                slot_pieces = slot.split("{{content}}")
                if slot_pieces[0]:
                    slot_items.append("'" + Template._jinja_escape(slot_pieces[0]) + "'")
                if len(slot_pieces) > 1:
                    slot_items.append(placeholder)
                    if slot_pieces[1]:
                        slot_items.append("'" + Template._jinja_escape(slot_pieces[1]) + "'")
            elif isinstance(slot, set):  # do not use {{ eos_token }} since it may be replaced
                if "bos_token" in slot and tokenizer.bos_token_id is not None:
                    slot_items.append("'" + tokenizer.bos_token + "'")
                elif "eos_token" in slot and tokenizer.eos_token_id is not None:
                    slot_items.append("'" + tokenizer.eos_token + "'")
            elif isinstance(slot, dict):
                raise ValueError("Dict is not supported.")

        return " + ".join(slot_items)

    def _get_jinja_template(self, tokenizer: "PreTrainedTokenizer") -> str:
chenych's avatar
chenych committed
238
        r"""Return the jinja template."""
chenych's avatar
chenych committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
        prefix = self._convert_slots_to_jinja(self.format_prefix.apply(), tokenizer)
        system = self._convert_slots_to_jinja(self.format_system.apply(), tokenizer, placeholder="system_message")
        user = self._convert_slots_to_jinja(self.format_user.apply(), tokenizer)
        assistant = self._convert_slots_to_jinja(self.format_assistant.apply(), tokenizer)
        jinja_template = ""
        if prefix:
            jinja_template += "{{ " + prefix + " }}"

        if self.default_system:
            jinja_template += "{% set system_message = '" + self._jinja_escape(self.default_system) + "' %}"

        jinja_template += (
            "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}"
            "{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}"
            "{% if system_message is defined %}{{ " + system + " }}{% endif %}"
            "{% for message in loop_messages %}"
            "{% set content = message['content'] %}"
            "{% if message['role'] == 'user' %}"
            "{{ " + user + " }}"
            "{% elif message['role'] == 'assistant' %}"
            "{{ " + assistant + " }}"
            "{% endif %}"
            "{% endfor %}"
        )
        return jinja_template

    def fix_jinja_template(self, tokenizer: "PreTrainedTokenizer") -> None:
chenych's avatar
chenych committed
266
        r"""Replace the jinja template in the tokenizer."""
chenych's avatar
chenych committed
267
268
269
270
271
272
273
274
275
276
        if tokenizer.chat_template is None or self.replace_jinja_template:
            try:
                tokenizer.chat_template = self._get_jinja_template(tokenizer)
            except ValueError as e:
                logger.info_rank0(f"Cannot add this chat template to tokenizer: {e}.")

    @staticmethod
    def _convert_slots_to_ollama(
        slots: "SLOTS", tokenizer: "PreTrainedTokenizer", placeholder: str = "content"
    ) -> str:
chenych's avatar
chenych committed
277
        r"""Convert slots to ollama template."""
chenych's avatar
chenych committed
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
        slot_items = []
        for slot in slots:
            if isinstance(slot, str):
                slot_pieces = slot.split("{{content}}")
                if slot_pieces[0]:
                    slot_items.append(slot_pieces[0])
                if len(slot_pieces) > 1:
                    slot_items.append("{{ " + placeholder + " }}")
                    if slot_pieces[1]:
                        slot_items.append(slot_pieces[1])
            elif isinstance(slot, set):  # do not use {{ eos_token }} since it may be replaced
                if "bos_token" in slot and tokenizer.bos_token_id is not None:
                    slot_items.append(tokenizer.bos_token)
                elif "eos_token" in slot and tokenizer.eos_token_id is not None:
                    slot_items.append(tokenizer.eos_token)
            elif isinstance(slot, dict):
                raise ValueError("Dict is not supported.")

        return "".join(slot_items)

    def _get_ollama_template(self, tokenizer: "PreTrainedTokenizer") -> str:
chenych's avatar
chenych committed
299
        r"""Return the ollama template."""
chenych's avatar
chenych committed
300
301
302
303
304
305
306
307
308
309
310
        prefix = self._convert_slots_to_ollama(self.format_prefix.apply(), tokenizer)
        system = self._convert_slots_to_ollama(self.format_system.apply(), tokenizer, placeholder=".System")
        user = self._convert_slots_to_ollama(self.format_user.apply(), tokenizer, placeholder=".Content")
        assistant = self._convert_slots_to_ollama(self.format_assistant.apply(), tokenizer, placeholder=".Content")
        return (
            f"{prefix}{{{{ if .System }}}}{system}{{{{ end }}}}"
            f"""{{{{ range .Messages }}}}{{{{ if eq .Role "user" }}}}{user}"""
            f"""{{{{ else if eq .Role "assistant" }}}}{assistant}{{{{ end }}}}{{{{ end }}}}"""
        )

    def get_ollama_modelfile(self, tokenizer: "PreTrainedTokenizer") -> str:
chenych's avatar
chenych committed
311
        r"""Return the ollama modelfile.
chenych's avatar
chenych committed
312
313
314
315
316
317
318
319
320
321
322
323
324
325

        TODO: support function calling.
        """
        modelfile = "# ollama modelfile auto-generated by llamafactory\n\n"
        modelfile += f'FROM .\n\nTEMPLATE """{self._get_ollama_template(tokenizer)}"""\n\n'

        if self.default_system:
            modelfile += f'SYSTEM """{self.default_system}"""\n\n'

        for stop_token_id in self.get_stop_token_ids(tokenizer):
            modelfile += f'PARAMETER stop "{tokenizer.convert_ids_to_tokens(stop_token_id)}"\n'

        modelfile += "PARAMETER num_ctx 4096\n"
        return modelfile
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
326
327
328
329


@dataclass
class Llama2Template(Template):
chenych's avatar
chenych committed
330
331
    r"""A template that fuse the system message to first user message."""

luopl's avatar
luopl committed
332
    @override
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
333
334
335
    def _encode(
        self,
        tokenizer: "PreTrainedTokenizer",
chenych's avatar
chenych committed
336
        messages: list[dict[str, str]],
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
337
338
        system: str,
        tools: str,
chenych's avatar
chenych committed
339
    ) -> list[list[int]]:
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
340
341
342
343
        system = system or self.default_system
        encoded_messages = []
        for i, message in enumerate(messages):
            elements = []
chenych's avatar
chenych committed
344

Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
345
            system_text = ""
chenych's avatar
chenych committed
346
347
348
349
350
351
            if i == 0:
                elements += self.format_prefix.apply()
                if system or tools:
                    tool_text = self.format_tools.apply(content=tools)[0] if tools else ""
                    system_text = self.format_system.apply(content=(system + tool_text))[0]

chenych's avatar
chenych committed
352
            if message["role"] == Role.USER:
chenych's avatar
chenych committed
353
                elements += self.format_user.apply(content=system_text + message["content"])
chenych's avatar
chenych committed
354
            elif message["role"] == Role.ASSISTANT:
chenych's avatar
chenych committed
355
                elements += self.format_assistant.apply(content=message["content"])
chenych's avatar
chenych committed
356
            elif message["role"] == Role.OBSERVATION:
chenych's avatar
chenych committed
357
                elements += self.format_observation.apply(content=message["content"])
chenych's avatar
chenych committed
358
            elif message["role"] == Role.FUNCTION:
chenych's avatar
chenych committed
359
                elements += self.format_function.apply(content=message["content"])
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
360
361
362
363
364
            else:
                raise NotImplementedError("Unexpected role: {}".format(message["role"]))

            encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements))

chenych's avatar
chenych committed
365
        return encoded_messages
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
366

chenych's avatar
chenych committed
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
    def _get_jinja_template(self, tokenizer: "PreTrainedTokenizer") -> str:
        prefix = self._convert_slots_to_jinja(self.format_prefix.apply(), tokenizer)
        system_message = self._convert_slots_to_jinja(
            self.format_system.apply(), tokenizer, placeholder="system_message"
        )
        user_message = self._convert_slots_to_jinja(self.format_user.apply(), tokenizer)
        assistant_message = self._convert_slots_to_jinja(self.format_assistant.apply(), tokenizer)
        jinja_template = ""
        if prefix:
            jinja_template += "{{ " + prefix + " }}"

        if self.default_system:
            jinja_template += "{% set system_message = '" + self._jinja_escape(self.default_system) + "' %}"

        jinja_template += (
            "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}"
            "{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}"
            "{% for message in loop_messages %}"
            "{% if loop.index0 == 0 and system_message is defined %}"
            "{% set content = " + system_message + " + message['content'] %}"
            "{% else %}{% set content = message['content'] %}{% endif %}"
            "{% if message['role'] == 'user' %}"
            "{{ " + user_message + " }}"
            "{% elif message['role'] == 'assistant' %}"
            "{{ " + assistant_message + " }}"
            "{% endif %}"
            "{% endfor %}"
        )
        return jinja_template

Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
397

chenych's avatar
chenych committed
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
@dataclass
class ReasoningTemplate(Template):
    r"""A template that add thought to assistant message."""

    @override
    def encode_oneturn(
        self,
        tokenizer: "PreTrainedTokenizer",
        messages: list[dict[str, str]],
        system: Optional[str] = None,
        tools: Optional[str] = None,
    ) -> tuple[list[int], list[int]]:
        messages = deepcopy(messages)
        for i in range(1, len(messages) - 2, 2):
            messages[i]["content"] = self.remove_thought(messages[i]["content"])

        if self.enable_thinking is False:  # remove all cot
            messages[-1]["content"] = self.remove_thought(messages[-1]["content"])

        prompt_ids, response_ids = super().encode_oneturn(tokenizer, messages, system, tools)
        if (
            self.thought_words[0] not in messages[-1]["content"]
            and self.thought_words[1] not in messages[-1]["content"]
        ):  # add empty cot
            if not self.enable_thinking:  # do not compute loss
                prompt_ids += self.get_thought_word_ids(tokenizer)
            else:  # do compute loss
                response_ids = self.get_thought_word_ids(tokenizer) + response_ids

        return prompt_ids, response_ids

    @override
    def encode_multiturn(
        self,
        tokenizer: "PreTrainedTokenizer",
        messages: list[dict[str, str]],
        system: Optional[str] = None,
        tools: Optional[str] = None,
    ) -> list[tuple[list[int], list[int]]]:
        messages = deepcopy(messages)
        if self.enable_thinking is False:  # remove all cot
            for i in range(1, len(messages), 2):
                messages[i]["content"] = self.remove_thought(messages[i]["content"])

        encoded_messages = self._encode(tokenizer, messages, system, tools)
        for i in range(0, len(messages), 2):
            if (
                self.thought_words[0] not in messages[i + 1]["content"]
                and self.thought_words[1] not in messages[i + 1]["content"]
            ):  # add empty cot
                if not self.enable_thinking:  # do not compute loss
                    encoded_messages[i] += self.get_thought_word_ids(tokenizer)
                else:  # do compute loss
                    encoded_messages[i + 1] = self.get_thought_word_ids(tokenizer) + encoded_messages[i + 1]

        return [(encoded_messages[i], encoded_messages[i + 1]) for i in range(0, len(encoded_messages), 2)]


chenych's avatar
chenych committed
456
TEMPLATES: dict[str, "Template"] = {}
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
457
458


chenych's avatar
chenych committed
459
def register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
460
461
462
463
464
465
466
    name: str,
    format_user: Optional["Formatter"] = None,
    format_assistant: Optional["Formatter"] = None,
    format_system: Optional["Formatter"] = None,
    format_function: Optional["Formatter"] = None,
    format_observation: Optional["Formatter"] = None,
    format_tools: Optional["Formatter"] = None,
chenych's avatar
chenych committed
467
    format_prefix: Optional["Formatter"] = None,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
468
    default_system: str = "",
chenych's avatar
chenych committed
469
470
    stop_words: Optional[list[str]] = None,
    thought_words: Optional[tuple[str, str]] = None,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
471
472
    efficient_eos: bool = False,
    replace_eos: bool = False,
luopl's avatar
luopl committed
473
    replace_jinja_template: bool = False,
chenych's avatar
chenych committed
474
    enable_thinking: Optional[bool] = True,
luopl's avatar
luopl committed
475
    mm_plugin: "BasePlugin" = get_mm_plugin(name="base"),
chenych's avatar
chenych committed
476
    template_class: type["Template"] = Template,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
477
) -> None:
chenych's avatar
chenych committed
478
    r"""Register a chat template.
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
479
480
481

    To add the following chat template:
    ```
luopl's avatar
luopl committed
482
483
484
485
    <s><user>user prompt here
    <model>model response here</s>
    <user>user prompt here
    <model>model response here</s>
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
486
487
488
489
    ```

    The corresponding code should be:
    ```
chenych's avatar
chenych committed
490
    register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
491
        name="custom",
luopl's avatar
luopl committed
492
493
494
        format_user=StringFormatter(slots=["<user>{{content}}\n<model>"]),
        format_assistant=StringFormatter(slots=["{{content}}</s>\n"]),
        format_prefix=EmptyFormatter("<s>"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
495
496
497
    )
    ```
    """
chenych's avatar
chenych committed
498
499
500
    if name in TEMPLATES:
        raise ValueError(f"Template {name} already exists.")

luopl's avatar
luopl committed
501
    default_slots = ["{{content}}"] if efficient_eos else ["{{content}}", {"eos_token"}]
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
502
    default_user_formatter = StringFormatter(slots=["{{content}}"])
luopl's avatar
luopl committed
503
    default_assistant_formatter = StringFormatter(slots=default_slots)
chenych's avatar
chenych committed
504
505
506
507
508
    if format_assistant is not None:
        default_function_formatter = FunctionFormatter(slots=format_assistant.slots, tool_format="default")
    else:
        default_function_formatter = FunctionFormatter(slots=default_slots, tool_format="default")

Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
509
    default_tool_formatter = ToolFormatter(tool_format="default")
chenych's avatar
chenych committed
510
511
    default_prefix_formatter = EmptyFormatter()
    TEMPLATES[name] = template_class(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
512
513
514
515
516
517
        format_user=format_user or default_user_formatter,
        format_assistant=format_assistant or default_assistant_formatter,
        format_system=format_system or default_user_formatter,
        format_function=format_function or default_function_formatter,
        format_observation=format_observation or format_user or default_user_formatter,
        format_tools=format_tools or default_tool_formatter,
chenych's avatar
chenych committed
518
        format_prefix=format_prefix or default_prefix_formatter,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
519
        default_system=default_system,
luopl's avatar
luopl committed
520
        stop_words=stop_words or [],
chenych's avatar
chenych committed
521
        thought_words=thought_words or ("<think>", "</think>"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
522
523
        efficient_eos=efficient_eos,
        replace_eos=replace_eos,
luopl's avatar
luopl committed
524
        replace_jinja_template=replace_jinja_template,
chenych's avatar
chenych committed
525
        enable_thinking=enable_thinking,
luopl's avatar
luopl committed
526
        mm_plugin=mm_plugin,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
527
528
529
    )


chenych's avatar
chenych committed
530
def parse_template(tokenizer: "PreTrainedTokenizer") -> "Template":
chenych's avatar
chenych committed
531
    r"""Extract a chat template from the tokenizer."""
chenych's avatar
chenych committed
532

chenych's avatar
chenych committed
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
    def find_diff(short_str: str, long_str: str) -> str:
        i, j = 0, 0
        diff = ""
        while i < len(short_str) and j < len(long_str):
            if short_str[i] == long_str[j]:
                i += 1
                j += 1
            else:
                diff += long_str[j]
                j += 1

        return diff

    prefix = tokenizer.decode(tokenizer.encode(""))

    messages = [{"role": "system", "content": "{{content}}"}]
    system_slot = tokenizer.apply_chat_template(messages, add_generation_prompt=False, tokenize=False)[len(prefix) :]

    messages = [{"role": "system", "content": ""}, {"role": "user", "content": "{{content}}"}]
    user_slot_empty_system = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
    user_slot_empty_system = user_slot_empty_system[len(prefix) :]

    messages = [{"role": "user", "content": "{{content}}"}]
    user_slot = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
    user_slot = user_slot[len(prefix) :]

    messages = [{"role": "user", "content": "{{content}}"}, {"role": "assistant", "content": "{{content}}"}]
    assistant_slot = tokenizer.apply_chat_template(messages, add_generation_prompt=False, tokenize=False)
    assistant_slot = assistant_slot[len(prefix) + len(user_slot) :]
chenych's avatar
chenych committed
562
    template_class = ReasoningTemplate if "<think>" in assistant_slot else Template
chenych's avatar
chenych committed
563
    assistant_slot = assistant_slot.replace("<think>", "").replace("</think>", "").lstrip("\n")  # remove thought tags
chenych's avatar
chenych committed
564
565
566
567
568
569
570
571

    if len(user_slot) > len(user_slot_empty_system):
        default_system = find_diff(user_slot_empty_system, user_slot)
        sole_system = system_slot.replace("{{content}}", default_system, 1)
        user_slot = user_slot[len(sole_system) :]
    else:  # if defaut_system is empty, user_slot_empty_system will be longer than user_slot
        default_system = ""

chenych's avatar
chenych committed
572
    return template_class(
chenych's avatar
chenych committed
573
574
575
576
577
578
579
580
581
582
583
584
585
        format_user=StringFormatter(slots=[user_slot]),
        format_assistant=StringFormatter(slots=[assistant_slot]),
        format_system=StringFormatter(slots=[system_slot]),
        format_function=FunctionFormatter(slots=[assistant_slot], tool_format="default"),
        format_observation=StringFormatter(slots=[user_slot]),
        format_tools=ToolFormatter(tool_format="default"),
        format_prefix=EmptyFormatter(slots=[prefix]) if prefix else EmptyFormatter(),
        default_system=default_system,
        stop_words=[],
        thought_words=("<think>", "</think>"),
        efficient_eos=False,
        replace_eos=False,
        replace_jinja_template=False,
chenych's avatar
chenych committed
586
        enable_thinking=True,
chenych's avatar
chenych committed
587
        mm_plugin=get_mm_plugin(name="base"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
588
589
590
    )


luopl's avatar
luopl committed
591
def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args: "DataArguments") -> "Template":
chenych's avatar
chenych committed
592
    r"""Get chat template and fixes the tokenizer."""
luopl's avatar
luopl committed
593
    if data_args.template is None:
chenych's avatar
chenych committed
594
595
596
597
598
599
        if isinstance(tokenizer.chat_template, str):
            logger.warning_rank0("`template` was not specified, try parsing the chat template from the tokenizer.")
            template = parse_template(tokenizer)
        else:
            logger.warning_rank0("`template` was not specified, use `empty` template.")
            template = TEMPLATES["empty"]  # placeholder
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
600
    else:
chenych's avatar
chenych committed
601
        if data_args.template not in TEMPLATES:
luopl's avatar
luopl committed
602
603
            raise ValueError(f"Template {data_args.template} does not exist.")

chenych's avatar
chenych committed
604
605
        template = TEMPLATES[data_args.template]

luopl's avatar
luopl committed
606
607
    if data_args.train_on_prompt and template.efficient_eos:
        raise ValueError("Current template does not support `train_on_prompt`.")
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
608

luopl's avatar
luopl committed
609
    if data_args.tool_format is not None:
luopl's avatar
luopl committed
610
        logger.info_rank0(f"Using tool format: {data_args.tool_format}.")
luopl's avatar
luopl committed
611
612
        default_slots = ["{{content}}"] if template.efficient_eos else ["{{content}}", {"eos_token"}]
        template.format_function = FunctionFormatter(slots=default_slots, tool_format=data_args.tool_format)
luopl's avatar
luopl committed
613
        template.format_tools = ToolFormatter(tool_format=data_args.tool_format)
chenych's avatar
chenych committed
614

chenych's avatar
chenych committed
615
616
617
618
619
    if data_args.default_system is not None:
        logger.info_rank0(f"Using default system message: {data_args.default_system}.")
        template.default_system = data_args.default_system

    template.enable_thinking = data_args.enable_thinking
chenych's avatar
chenych committed
620
621
    template.fix_special_tokens(tokenizer)
    template.fix_jinja_template(tokenizer)
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
622
623
624
    return template


chenych's avatar
chenych committed
625
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
626
627
    name="alpaca",
    format_user=StringFormatter(slots=["### Instruction:\n{{content}}\n\n### Response:\n"]),
luopl's avatar
luopl committed
628
    format_assistant=StringFormatter(slots=["{{content}}", {"eos_token"}, "\n\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
629
    default_system=(
luopl's avatar
luopl committed
630
        "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n"
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
631
    ),
luopl's avatar
luopl committed
632
    replace_jinja_template=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
633
634
635
)


chenych's avatar
chenych committed
636
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
637
638
    name="aquila",
    format_user=StringFormatter(slots=["Human: {{content}}###Assistant:"]),
luopl's avatar
luopl committed
639
640
    format_assistant=StringFormatter(slots=["{{content}}###"]),
    format_system=StringFormatter(slots=["System: {{content}}###"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
641
642
643
644
645
646
647
648
    default_system=(
        "A chat between a curious human and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the human's questions."
    ),
    stop_words=["</s>"],
)


chenych's avatar
chenych committed
649
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
650
651
652
653
654
655
656
657
    name="atom",
    format_user=StringFormatter(
        slots=[{"bos_token"}, "Human: {{content}}\n", {"eos_token"}, {"bos_token"}, "Assistant:"]
    ),
    format_assistant=StringFormatter(slots=["{{content}}\n", {"eos_token"}]),
)


chenych's avatar
chenych committed
658
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
659
660
661
662
663
664
    name="baichuan",
    format_user=StringFormatter(slots=[{"token": "<reserved_102>"}, "{{content}}", {"token": "<reserved_103>"}]),
    efficient_eos=True,
)


chenych's avatar
chenych committed
665
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
666
667
668
669
670
671
    name="baichuan2",
    format_user=StringFormatter(slots=["<reserved_106>{{content}}<reserved_107>"]),
    efficient_eos=True,
)


chenych's avatar
chenych committed
672
673
674
675
676
677
678
679
680
681
682
register_template(
    name="bailing",
    format_user=StringFormatter(slots=["<role>HUMAN</role>{{content}}<role>ASSISTANT</role>"]),
    format_system=StringFormatter(slots=["<role>SYSTEM</role>{{content}}"]),
    format_observation=StringFormatter(slots=["<role>OBSERVATION</role>{{content}}<role>ASSISTANT</role>"]),
    stop_words=["<|endoftext|>"],
    efficient_eos=True,
)


register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
683
684
    name="belle",
    format_user=StringFormatter(slots=["Human: {{content}}\n\nBelle: "]),
luopl's avatar
luopl committed
685
    format_assistant=StringFormatter(slots=["{{content}}", {"eos_token"}, "\n\n"]),
chenych's avatar
chenych committed
686
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
687
688
689
)


chenych's avatar
chenych committed
690
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
691
692
693
694
695
    name="bluelm",
    format_user=StringFormatter(slots=[{"token": "[|Human|]:"}, "{{content}}", {"token": "[|AI|]:"}]),
)


chenych's avatar
chenych committed
696
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
697
698
    name="breeze",
    format_user=StringFormatter(slots=["[INST] {{content}} [/INST] "]),
chenych's avatar
chenych committed
699
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
700
701
702
703
    efficient_eos=True,
)


chenych's avatar
chenych committed
704
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
705
706
    name="chatglm2",
    format_user=StringFormatter(slots=["[Round {{idx}}]\n\n问:{{content}}\n\n答:"]),
chenych's avatar
chenych committed
707
    format_prefix=EmptyFormatter(slots=[{"token": "[gMASK]"}, {"token": "sop"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
708
709
710
711
    efficient_eos=True,
)


chenych's avatar
chenych committed
712
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
713
714
715
    name="chatglm3",
    format_user=StringFormatter(slots=[{"token": "<|user|>"}, "\n", "{{content}}", {"token": "<|assistant|>"}]),
    format_assistant=StringFormatter(slots=["\n", "{{content}}"]),
chenych's avatar
chenych committed
716
    format_system=StringFormatter(slots=[{"token": "<|system|>"}, "\n", "{{content}}"]),
luopl's avatar
luopl committed
717
    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="glm4"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
718
719
720
    format_observation=StringFormatter(
        slots=[{"token": "<|observation|>"}, "\n", "{{content}}", {"token": "<|assistant|>"}]
    ),
chenych's avatar
chenych committed
721
722
    format_tools=ToolFormatter(tool_format="glm4"),
    format_prefix=EmptyFormatter(slots=[{"token": "[gMASK]"}, {"token": "sop"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
723
724
725
726
727
    stop_words=["<|user|>", "<|observation|>"],
    efficient_eos=True,
)


chenych's avatar
chenych committed
728
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
729
730
    name="chatml",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
731
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
732
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
chenych's avatar
chenych committed
733
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
734
735
    stop_words=["<|im_end|>", "<|im_start|>"],
    replace_eos=True,
luopl's avatar
luopl committed
736
    replace_jinja_template=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
737
738
739
)


luopl's avatar
luopl committed
740
# copied from chatml template
chenych's avatar
chenych committed
741
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
742
743
    name="chatml_de",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
744
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
745
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
chenych's avatar
chenych committed
746
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
747
748
749
    default_system="Du bist ein freundlicher und hilfsbereiter KI-Assistent.",
    stop_words=["<|im_end|>", "<|im_start|>"],
    replace_eos=True,
luopl's avatar
luopl committed
750
    replace_jinja_template=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
751
752
753
)


chenych's avatar
chenych committed
754
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
755
    name="codegeex2",
chenych's avatar
chenych committed
756
757
758
759
    format_prefix=EmptyFormatter(slots=[{"token": "[gMASK]"}, {"token": "sop"}]),
)


chenych's avatar
chenych committed
760
register_template(
chenych's avatar
chenych committed
761
762
763
    name="codegeex4",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|assistant|>\n"]),
    format_system=StringFormatter(slots=["<|system|>\n{{content}}"]),
luopl's avatar
luopl committed
764
    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="glm4"),
chenych's avatar
chenych committed
765
766
767
768
769
770
771
772
773
    format_observation=StringFormatter(slots=["<|observation|>\n{{content}}<|assistant|>\n"]),
    format_tools=ToolFormatter(tool_format="glm4"),
    format_prefix=EmptyFormatter(slots=["[gMASK]<sop>"]),
    default_system=(
        "你是一位智能编程助手,你叫CodeGeeX。你会为用户回答关于编程、代码、计算机方面的任何问题,"
        "并提供格式规范、可以执行、准确安全的代码,并在必要时提供详细的解释。"
    ),
    stop_words=["<|user|>", "<|observation|>"],
    efficient_eos=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
774
775
776
)


chenych's avatar
chenych committed
777
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
778
779
780
781
782
783
784
785
786
    name="cohere",
    format_user=StringFormatter(
        slots=[
            (
                "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{content}}<|END_OF_TURN_TOKEN|>"
                "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
            )
        ]
    ),
chenych's avatar
chenych committed
787
788
    format_system=StringFormatter(slots=["<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{content}}<|END_OF_TURN_TOKEN|>"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
789
790
791
)


chenych's avatar
chenych committed
792
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
793
794
    name="cpm",
    format_user=StringFormatter(slots=["<用户>{{content}}<AI>"]),
chenych's avatar
chenych committed
795
796
797
798
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
)


luopl's avatar
luopl committed
799
# copied from chatml template
chenych's avatar
chenych committed
800
register_template(
luopl's avatar
luopl committed
801
802
    name="cpm3",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
803
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
804
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
chenych's avatar
chenych committed
805
806
807
808
809
810
811
812
813
814
815
816
817
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    stop_words=["<|im_end|>"],
)


# copied from chatml template
register_template(
    name="cpm4",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
818
819
820
821
822
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    stop_words=["<|im_end|>"],
)


luopl's avatar
luopl committed
823
# copied from chatml template
chenych's avatar
chenych committed
824
register_template(
chenych's avatar
chenych committed
825
826
    name="dbrx",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
827
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
chenych's avatar
chenych committed
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    default_system=(
        "You are DBRX, created by Databricks. You were last updated in December 2023. "
        "You answer questions based on information available up to that point.\n"
        "YOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough "
        "responses to more complex and open-ended questions.\nYou assist with various tasks, "
        "from writing to coding (using markdown for code blocks — remember to use ``` with "
        "code, JSON, and tables).\n(You do not have real-time data access or code execution "
        "capabilities. You avoid stereotyping and provide balanced perspectives on "
        "controversial topics. You do not provide song lyrics, poems, or news articles and "
        "do not divulge details of your training data.)\nThis is your system prompt, "
        "guiding your responses. Do not reference it, just respond to the user. If you find "
        "yourself talking about this message, stop. You should be responding appropriately "
        "and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION "
        "ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER'S QUERY."
    ),
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
846
    replace_eos=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
847
848
849
)


chenych's avatar
chenych committed
850
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
851
852
    name="deepseek",
    format_user=StringFormatter(slots=["User: {{content}}\n\nAssistant:"]),
chenych's avatar
chenych committed
853
854
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
855
856
857
)


chenych's avatar
chenych committed
858
register_template(
luopl's avatar
luopl committed
859
860
861
862
863
864
    name="deepseek3",
    format_user=StringFormatter(slots=["<|User|>{{content}}<|Assistant|>"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
)


chenych's avatar
chenych committed
865
866
867
868
869
870
871
872
873
# copied from deepseek3 template
register_template(
    name="deepseekr1",
    format_user=StringFormatter(slots=["<|User|>{{content}}<|Assistant|>"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    template_class=ReasoningTemplate,
)


chenych's avatar
chenych committed
874
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
875
876
    name="deepseekcoder",
    format_user=StringFormatter(slots=["### Instruction:\n{{content}}\n### Response:"]),
luopl's avatar
luopl committed
877
    format_assistant=StringFormatter(slots=["\n{{content}}\n<|EOT|>\n"]),
chenych's avatar
chenych committed
878
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
879
    default_system=(
chenych's avatar
chenych committed
880
881
        "You are an AI programming assistant, utilizing the DeepSeek Coder model, "
        "developed by DeepSeek Company, and you only answer questions related to computer science. "
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
882
        "For politically sensitive questions, security and privacy issues, "
chenych's avatar
chenych committed
883
        "and other non-computer science questions, you will refuse to answer.\n"
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
884
885
886
887
    ),
)


chenych's avatar
chenych committed
888
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
889
    name="default",
chenych's avatar
chenych committed
890
    format_user=StringFormatter(slots=["Human: {{content}}", {"eos_token"}, "\nAssistant:"]),
luopl's avatar
luopl committed
891
    format_assistant=StringFormatter(slots=["{{content}}", {"eos_token"}, "\n"]),
chenych's avatar
chenych committed
892
893
    format_system=StringFormatter(slots=["System: {{content}}", {"eos_token"}, "\n"]),
    replace_jinja_template=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
894
895
896
)


chenych's avatar
chenych committed
897
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
898
    name="empty",
chenych's avatar
chenych committed
899
    format_assistant=StringFormatter(slots=["{{content}}"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
900
901
902
)


chenych's avatar
chenych committed
903
register_template(
luopl's avatar
luopl committed
904
905
    name="exaone",
    format_user=StringFormatter(slots=["[|user|]{{content}}\n[|assistant|]"]),
luopl's avatar
luopl committed
906
    format_assistant=StringFormatter(slots=["{{content}}", {"eos_token"}, "\n"]),
luopl's avatar
luopl committed
907
908
909
910
    format_system=StringFormatter(slots=["[|system|]{{content}}[|endofturn|]\n"]),
)


chenych's avatar
chenych committed
911
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
912
913
    name="falcon",
    format_user=StringFormatter(slots=["User: {{content}}\nFalcon:"]),
luopl's avatar
luopl committed
914
    format_assistant=StringFormatter(slots=["{{content}}\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
915
916
917
918
    efficient_eos=True,
)


chenych's avatar
chenych committed
919
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
920
    name="fewshot",
luopl's avatar
luopl committed
921
    format_assistant=StringFormatter(slots=["{{content}}\n\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
922
    efficient_eos=True,
chenych's avatar
chenych committed
923
    replace_jinja_template=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
924
925
926
)


chenych's avatar
chenych committed
927
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
928
929
    name="gemma",
    format_user=StringFormatter(slots=["<start_of_turn>user\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]),
luopl's avatar
luopl committed
930
    format_assistant=StringFormatter(slots=["{{content}}<end_of_turn>\n"]),
chenych's avatar
chenych committed
931
932
933
934
935
936
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
    format_observation=StringFormatter(
        slots=["<start_of_turn>tool\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]
    ),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    stop_words=["<end_of_turn>"],
chenych's avatar
chenych committed
937
    replace_eos=True,
chenych's avatar
chenych committed
938
939
940
941
942
943
944
945
946
947
    template_class=Llama2Template,
)


# copied from gemma template
register_template(
    name="gemma3",
    format_user=StringFormatter(slots=["<start_of_turn>user\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<end_of_turn>\n"]),
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
chenych's avatar
chenych committed
948
949
950
951
    format_observation=StringFormatter(
        slots=["<start_of_turn>tool\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]
    ),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
chenych's avatar
chenych committed
952
    stop_words=["<end_of_turn>"],
chenych's avatar
chenych committed
953
    replace_eos=True,
chenych's avatar
chenych committed
954
955
    mm_plugin=get_mm_plugin("gemma3", image_token="<image_soft_token>"),
    template_class=Llama2Template,
chenych's avatar
chenych committed
956
957
958
)


chenych's avatar
chenych committed
959
register_template(
chenych's avatar
chenych committed
960
961
962
963
    name="glm4",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|assistant|>"]),
    format_assistant=StringFormatter(slots=["\n{{content}}"]),
    format_system=StringFormatter(slots=["<|system|>\n{{content}}"]),
luopl's avatar
luopl committed
964
    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="glm4"),
chenych's avatar
chenych committed
965
966
967
968
    format_observation=StringFormatter(slots=["<|observation|>\n{{content}}<|assistant|>"]),
    format_tools=ToolFormatter(tool_format="glm4"),
    format_prefix=EmptyFormatter(slots=["[gMASK]<sop>"]),
    stop_words=["<|user|>", "<|observation|>"],
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
969
970
971
972
    efficient_eos=True,
)


chenych's avatar
chenych committed
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
# copied from glm4 template
register_template(
    name="glmz1",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|assistant|>"]),
    format_assistant=StringFormatter(slots=["\n{{content}}"]),
    format_system=StringFormatter(slots=["<|system|>\n{{content}}"]),
    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="glm4"),
    format_observation=StringFormatter(slots=["<|observation|>\n{{content}}<|assistant|>"]),
    format_tools=ToolFormatter(tool_format="glm4"),
    format_prefix=EmptyFormatter(slots=["[gMASK]<sop>"]),
    stop_words=["<|user|>", "<|observation|>"],
    efficient_eos=True,
    template_class=ReasoningTemplate,
)


chenych's avatar
chenych committed
989
register_template(
luopl's avatar
luopl committed
990
991
992
993
994
995
996
997
998
999
1000
    name="granite3",
    format_user=StringFormatter(
        slots=[
            "<|start_of_role|>user<|end_of_role|>{{content}}<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>"
        ]
    ),
    format_assistant=StringFormatter(slots=["{{content}}<|end_of_text|>\n"]),
    format_system=StringFormatter(slots=["<|start_of_role|>system<|end_of_role|>{{content}}<|end_of_text|>\n"]),
)


chenych's avatar
chenych committed
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
register_template(
    name="granite3_vision",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}\n<|assistant|>\n"]),
    format_system=StringFormatter(slots=["<|system|>\n{{content}}\n"]),
    default_system=(
        "A chat between a curious user and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the user's questions."
    ),
    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
)


chenych's avatar
chenych committed
1013
register_template(
luopl's avatar
luopl committed
1014
1015
1016
1017
1018
1019
1020
    name="index",
    format_user=StringFormatter(slots=["reserved_0{{content}}reserved_1"]),
    format_system=StringFormatter(slots=["<unk>{{content}}"]),
    efficient_eos=True,
)


chenych's avatar
chenych committed
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
register_template(
    name="hunyuan",
    format_user=StringFormatter(slots=["<|bos|>user\n{{content}}<|eos|>\n<|bos|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|eos|>\n"]),
    format_system=StringFormatter(slots=["<|bos|>system\n{{content}}<|eos|>\n"]),
    format_prefix=EmptyFormatter(slots=["<|bos|>"]),
    stop_words=["<|eos|>"],
)


chenych's avatar
chenych committed
1031
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1032
    name="intern",
chenych's avatar
chenych committed
1033
    format_user=StringFormatter(slots=["<|User|>:{{content}}\n<|Bot|>:"]),
luopl's avatar
luopl committed
1034
    format_assistant=StringFormatter(slots=["{{content}}<eoa>\n"]),
chenych's avatar
chenych committed
1035
1036
    format_system=StringFormatter(slots=["<|System|>:{{content}}\n"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
chenych's avatar
chenych committed
1037
1038
1039
1040
1041
1042
1043
    default_system=(
        "You are an AI assistant whose name is InternLM (书生·浦语).\n"
        "- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory "
        "(上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n"
        "- InternLM (书生·浦语) can understand and communicate fluently in the language "
        "chosen by the user such as English and 中文."
    ),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1044
1045
1046
1047
    stop_words=["<eoa>"],
)


chenych's avatar
chenych committed
1048
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1049
1050
    name="intern2",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1051
1052
1053
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
chenych's avatar
chenych committed
1054
1055
1056
1057
1058
1059
1060
    default_system=(
        "You are an AI assistant whose name is InternLM (书生·浦语).\n"
        "- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory "
        "(上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n"
        "- InternLM (书生·浦语) can understand and communicate fluently in the language "
        "chosen by the user such as English and 中文."
    ),
luopl's avatar
luopl committed
1061
1062
1063
1064
    stop_words=["<|im_end|>"],
)


chenych's avatar
chenych committed
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
register_template(
    name="intern_vl",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    default_system=(
        "你是书生·万象,英文名是InternVL,是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。"
    ),
    stop_words=["<|im_end|>"],
    mm_plugin=get_mm_plugin(name="intern_vl", image_token="<image>", video_token="<video>"),
)


chenych's avatar
chenych committed
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
register_template(
    name="kimi_vl",
    format_user=StringFormatter(
        slots=["<|im_user|>user<|im_middle|>{{content}}<|im_end|><|im_assistant|>assistant<|im_middle|>"]
    ),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>"]),
    format_system=StringFormatter(slots=["<|im_system|>system<|im_middle|>{{content}}<|im_end|>"]),
    default_system="You are a helpful assistant",
    stop_words=["<|im_end|>"],
    thought_words=("◁think▷", "◁/think▷"),
    mm_plugin=get_mm_plugin("kimi_vl", image_token="<|media_pad|>"),
chenych's avatar
chenych committed
1090
    template_class=ReasoningTemplate,
chenych's avatar
chenych committed
1091
1092
1093
)


chenych's avatar
chenych committed
1094
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1095
1096
1097
    name="llama2",
    format_user=StringFormatter(slots=[{"bos_token"}, "[INST] {{content}} [/INST]"]),
    format_system=StringFormatter(slots=["<<SYS>>\n{{content}}\n<</SYS>>\n\n"]),
chenych's avatar
chenych committed
1098
    template_class=Llama2Template,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1099
1100
1101
)


luopl's avatar
luopl committed
1102
# copied from llama2 template
chenych's avatar
chenych committed
1103
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1104
1105
1106
1107
    name="llama2_zh",
    format_user=StringFormatter(slots=[{"bos_token"}, "[INST] {{content}} [/INST]"]),
    format_system=StringFormatter(slots=["<<SYS>>\n{{content}}\n<</SYS>>\n\n"]),
    default_system="You are a helpful assistant. 你是一个乐于助人的助手。",
chenych's avatar
chenych committed
1108
    template_class=Llama2Template,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1109
1110
1111
)


chenych's avatar
chenych committed
1112
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1113
1114
1115
1116
1117
1118
1119
1120
1121
    name="llama3",
    format_user=StringFormatter(
        slots=[
            (
                "<|start_header_id|>user<|end_header_id|>\n\n{{content}}<|eot_id|>"
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
luopl's avatar
luopl committed
1122
    format_assistant=StringFormatter(slots=["{{content}}<|eot_id|>"]),
chenych's avatar
chenych committed
1123
    format_system=StringFormatter(slots=["<|start_header_id|>system<|end_header_id|>\n\n{{content}}<|eot_id|>"]),
luopl's avatar
luopl committed
1124
    format_function=FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3"),
chenych's avatar
chenych committed
1125
1126
1127
    format_observation=StringFormatter(
        slots=[
            (
luopl's avatar
luopl committed
1128
                "<|start_header_id|>ipython<|end_header_id|>\n\n{{content}}<|eot_id|>"
chenych's avatar
chenych committed
1129
1130
1131
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1132
    ),
luopl's avatar
luopl committed
1133
    format_tools=ToolFormatter(tool_format="llama3"),
chenych's avatar
chenych committed
1134
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
luopl's avatar
luopl committed
1135
    stop_words=["<|eot_id|>", "<|eom_id|>"],
chenych's avatar
chenych committed
1136
    replace_eos=True,
luopl's avatar
luopl committed
1137
1138
1139
)


chenych's avatar
chenych committed
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
register_template(
    name="llama4",
    format_user=StringFormatter(
        slots=["<|header_start|>user<|header_end|>\n\n{{content}}<|eot|><|header_start|>assistant<|header_end|>\n\n"]
    ),
    format_assistant=StringFormatter(slots=["{{content}}<|eot|>"]),
    format_system=StringFormatter(slots=["<|header_start|>system<|header_end|>\n\n{{content}}<|eot|>"]),
    format_function=FunctionFormatter(slots=["{{content}}<|eot|>"], tool_format="llama3"),
    format_observation=StringFormatter(
        slots=[
            "<|header_start|>ipython<|header_end|>\n\n{{content}}<|eot|><|header_start|>assistant<|header_end|>\n\n"
        ]
    ),
    format_tools=ToolFormatter(tool_format="llama3"),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    stop_words=["<|eot|>", "<|eom|>"],
chenych's avatar
chenych committed
1156
    replace_eos=True,
chenych's avatar
chenych committed
1157
1158
1159
1160
    mm_plugin=get_mm_plugin(name="llama4", image_token="<|image|>"),
)


luopl's avatar
luopl committed
1161
# copied from llama3 template
chenych's avatar
chenych committed
1162
register_template(
luopl's avatar
luopl committed
1163
1164
1165
1166
1167
1168
1169
1170
1171
    name="mllama",
    format_user=StringFormatter(
        slots=[
            (
                "<|start_header_id|>user<|end_header_id|>\n\n{{content}}<|eot_id|>"
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
luopl's avatar
luopl committed
1172
    format_assistant=StringFormatter(slots=["{{content}}<|eot_id|>"]),
luopl's avatar
luopl committed
1173
    format_system=StringFormatter(slots=["<|start_header_id|>system<|end_header_id|>\n\n{{content}}<|eot_id|>"]),
luopl's avatar
luopl committed
1174
    format_function=FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3"),
luopl's avatar
luopl committed
1175
1176
1177
    format_observation=StringFormatter(
        slots=[
            (
luopl's avatar
luopl committed
1178
                "<|start_header_id|>ipython<|end_header_id|>\n\n{{content}}<|eot_id|>"
luopl's avatar
luopl committed
1179
1180
1181
1182
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
luopl's avatar
luopl committed
1183
    format_tools=ToolFormatter(tool_format="llama3"),
luopl's avatar
luopl committed
1184
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
luopl's avatar
luopl committed
1185
    stop_words=["<|eot_id|>", "<|eom_id|>"],
chenych's avatar
chenych committed
1186
    replace_eos=True,
luopl's avatar
luopl committed
1187
1188
1189
1190
    mm_plugin=get_mm_plugin(name="mllama", image_token="<|image|>"),
)


chenych's avatar
chenych committed
1191
1192
1193
1194
1195
1196
1197
1198
1199
register_template(
    name="moonlight",
    format_user=StringFormatter(
        slots=["<|im_user|>user<|im_middle|>{{content}}<|im_end|><|im_assistant|>assistant<|im_middle|>"]
    ),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>"]),
    format_system=StringFormatter(slots=["<|im_system|>system<|im_middle|>{{content}}<|im_end|>"]),
    default_system="You are a helpful assistant provided by Moonshot-AI.",
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1200
    replace_eos=True,
chenych's avatar
chenych committed
1201
1202
1203
)


luopl's avatar
luopl committed
1204
# copied from vicuna template
chenych's avatar
chenych committed
1205
register_template(
luopl's avatar
luopl committed
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
    name="llava",
    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
    default_system=(
        "A chat between a curious user and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the user's questions."
    ),
    mm_plugin=get_mm_plugin(name="llava", image_token="<image>"),
)


luopl's avatar
luopl committed
1216
# copied from vicuna template
chenych's avatar
chenych committed
1217
register_template(
luopl's avatar
luopl committed
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
    name="llava_next",
    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
    default_system=(
        "A chat between a curious user and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the user's questions."
    ),
    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
)


luopl's avatar
luopl committed
1228
# copied from llama3 template
chenych's avatar
chenych committed
1229
register_template(
luopl's avatar
luopl committed
1230
1231
1232
1233
1234
1235
1236
1237
1238
    name="llava_next_llama3",
    format_user=StringFormatter(
        slots=[
            (
                "<|start_header_id|>user<|end_header_id|>\n\n{{content}}<|eot_id|>"
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
luopl's avatar
luopl committed
1239
    format_assistant=StringFormatter(slots=["{{content}}<|eot_id|>"]),
luopl's avatar
luopl committed
1240
    format_system=StringFormatter(slots=["<|start_header_id|>system<|end_header_id|>\n\n{{content}}<|eot_id|>"]),
luopl's avatar
luopl committed
1241
    format_function=FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3"),
luopl's avatar
luopl committed
1242
1243
1244
    format_observation=StringFormatter(
        slots=[
            (
luopl's avatar
luopl committed
1245
                "<|start_header_id|>ipython<|end_header_id|>\n\n{{content}}<|eot_id|>"
luopl's avatar
luopl committed
1246
1247
1248
1249
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
luopl's avatar
luopl committed
1250
    format_tools=ToolFormatter(tool_format="llama3"),
luopl's avatar
luopl committed
1251
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
luopl's avatar
luopl committed
1252
    stop_words=["<|eot_id|>", "<|eom_id|>"],
chenych's avatar
chenych committed
1253
    replace_eos=True,
luopl's avatar
luopl committed
1254
1255
1256
1257
    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
)


luopl's avatar
luopl committed
1258
# copied from mistral template
chenych's avatar
chenych committed
1259
register_template(
luopl's avatar
luopl committed
1260
    name="llava_next_mistral",
luopl's avatar
luopl committed
1261
1262
1263
    format_user=StringFormatter(slots=["[INST] {{content}}[/INST]"]),
    format_assistant=StringFormatter(slots=[" {{content}}", {"eos_token"}]),
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
chenych's avatar
chenych committed
1264
    format_function=FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", {"eos_token"}], tool_format="mistral"),
luopl's avatar
luopl committed
1265
1266
    format_observation=StringFormatter(slots=["""[TOOL_RESULTS] {"content": {{content}}}[/TOOL_RESULTS]"""]),
    format_tools=ToolFormatter(tool_format="mistral"),
luopl's avatar
luopl committed
1267
1268
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
chenych's avatar
chenych committed
1269
    template_class=Llama2Template,
luopl's avatar
luopl committed
1270
1271
1272
)


chenych's avatar
chenych committed
1273
1274
# copied from qwen template
register_template(
luopl's avatar
luopl committed
1275
1276
    name="llava_next_qwen",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1277
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1278
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1279
1280
1281
1282
1283
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
luopl's avatar
luopl committed
1284
1285
    default_system="You are a helpful assistant.",
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1286
    replace_eos=True,
luopl's avatar
luopl committed
1287
1288
1289
1290
    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
)


luopl's avatar
luopl committed
1291
# copied from chatml template
chenych's avatar
chenych committed
1292
register_template(
luopl's avatar
luopl committed
1293
1294
    name="llava_next_yi",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1295
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1296
1297
1298
1299
1300
1301
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    stop_words=["<|im_end|>"],
    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
)


luopl's avatar
luopl committed
1302
# copied from vicuna template
chenych's avatar
chenych committed
1303
register_template(
luopl's avatar
luopl committed
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
    name="llava_next_video",
    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
    default_system=(
        "A chat between a curious user and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the user's questions."
    ),
    mm_plugin=get_mm_plugin(name="llava_next_video", image_token="<image>", video_token="<video>"),
)


luopl's avatar
luopl committed
1314
# copied from mistral template
chenych's avatar
chenych committed
1315
register_template(
luopl's avatar
luopl committed
1316
    name="llava_next_video_mistral",
luopl's avatar
luopl committed
1317
1318
1319
    format_user=StringFormatter(slots=["[INST] {{content}}[/INST]"]),
    format_assistant=StringFormatter(slots=[" {{content}}", {"eos_token"}]),
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
chenych's avatar
chenych committed
1320
    format_function=FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", {"eos_token"}], tool_format="mistral"),
luopl's avatar
luopl committed
1321
1322
    format_observation=StringFormatter(slots=["""[TOOL_RESULTS] {"content": {{content}}}[/TOOL_RESULTS]"""]),
    format_tools=ToolFormatter(tool_format="mistral"),
luopl's avatar
luopl committed
1323
1324
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    mm_plugin=get_mm_plugin(name="llava_next_video", image_token="<image>", video_token="<video>"),
chenych's avatar
chenych committed
1325
    template_class=Llama2Template,
luopl's avatar
luopl committed
1326
1327
1328
)


luopl's avatar
luopl committed
1329
# copied from chatml template
chenych's avatar
chenych committed
1330
register_template(
luopl's avatar
luopl committed
1331
1332
    name="llava_next_video_yi",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1333
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1334
1335
1336
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    stop_words=["<|im_end|>"],
    mm_plugin=get_mm_plugin(name="llava_next_video", image_token="<image>", video_token="<video>"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1337
1338
1339
)


luopl's avatar
luopl committed
1340
# copied from chatml template
chenych's avatar
chenych committed
1341
register_template(
luopl's avatar
luopl committed
1342
1343
1344
1345
1346
1347
    name="marco",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    default_system=(
chenych's avatar
chenych committed
1348
1349
        "你是一个经过良好训练的AI助手,你的名字是Marco-o1."
        "由阿里国际数字商业集团的AI Business创造.\n## 重要!!!!!\n"
luopl's avatar
luopl committed
1350
1351
1352
1353
1354
1355
1356
        "当你回答问题时,你的思考应该在<Thought>内完成,<Output>内输出你的结果。\n"
        "<Thought>应该尽可能是英文,但是有2个特例,一个是对原文中的引用,另一个是是数学应该使用markdown格式,<Output>内的输出需要遵循用户输入的语言。\n"
    ),
    stop_words=["<|im_end|>"],
)


chenych's avatar
chenych committed
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
# copied from qwen template
register_template(
    name="mimo",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
    default_system="You are a helpful assistant.",
    stop_words=["<|im_end|>"],
    replace_eos=True,
    template_class=ReasoningTemplate,
)

# copied from qwen2vl
register_template(
    name="mimo_vl",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
    default_system="You are MiMo, an AI assistant developed by Xiaomi.",
    stop_words=["<|im_end|>"],
    replace_eos=True,
    mm_plugin=get_mm_plugin(name="qwen2_vl", image_token="<|image_pad|>", video_token="<|video_pad|>"),
    template_class=ReasoningTemplate,
)


luopl's avatar
luopl committed
1393
# copied from chatml template
chenych's avatar
chenych committed
1394
register_template(
luopl's avatar
luopl committed
1395
1396
1397
1398
1399
    name="minicpm_v",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1400
    default_system="You are a helpful assistant.",
luopl's avatar
luopl committed
1401
1402
1403
1404
    mm_plugin=get_mm_plugin(name="minicpm_v", image_token="<image>", video_token="<video>"),
)


chenych's avatar
chenych committed
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
# copied from minicpm_v template
register_template(
    name="minicpm_o",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    stop_words=["<|im_end|>"],
    default_system="You are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
    mm_plugin=get_mm_plugin(name="minicpm_v", image_token="<image>", video_token="<video>", audio_token="<audio>"),
)


# mistral tokenizer v3 tekken
register_template(
    name="ministral",
    format_user=StringFormatter(slots=["[INST]{{content}}[/INST]"]),
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
    format_function=FunctionFormatter(slots=["[TOOL_CALLS]{{content}}", {"eos_token"}], tool_format="mistral"),
    format_observation=StringFormatter(slots=["""[TOOL_RESULTS]{"content": {{content}}}[/TOOL_RESULTS]"""]),
    format_tools=ToolFormatter(tool_format="mistral"),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    template_class=Llama2Template,
)


# mistral tokenizer v3
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1432
    name="mistral",
luopl's avatar
luopl committed
1433
1434
1435
    format_user=StringFormatter(slots=["[INST] {{content}}[/INST]"]),
    format_assistant=StringFormatter(slots=[" {{content}}", {"eos_token"}]),
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
chenych's avatar
chenych committed
1436
    format_function=FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", {"eos_token"}], tool_format="mistral"),
luopl's avatar
luopl committed
1437
1438
    format_observation=StringFormatter(slots=["""[TOOL_RESULTS] {"content": {{content}}}[/TOOL_RESULTS]"""]),
    format_tools=ToolFormatter(tool_format="mistral"),
chenych's avatar
chenych committed
1439
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
chenych's avatar
chenych committed
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
    template_class=Llama2Template,
)


# mistral tokenizer v7 tekken (copied from ministral)
register_template(
    name="mistral_small",
    format_user=StringFormatter(slots=["[INST]{{content}}[/INST]"]),
    format_system=StringFormatter(slots=["[SYSTEM_PROMPT]{{content}}[/SYSTEM_PROMPT]"]),
    format_function=FunctionFormatter(slots=["[TOOL_CALLS]{{content}}", {"eos_token"}], tool_format="mistral"),
    format_observation=StringFormatter(slots=["""[TOOL_RESULTS]{"content": {{content}}}[/TOOL_RESULTS]"""]),
    format_tools=ToolFormatter(tool_format="mistral"),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
chenych's avatar
chenych committed
1453
    mm_plugin=get_mm_plugin(name="pixtral", image_token="[IMG]"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1454
1455
1456
)


chenych's avatar
chenych committed
1457
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1458
    name="olmo",
chenych's avatar
chenych committed
1459
1460
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|assistant|>\n"]),
    format_prefix=EmptyFormatter(slots=[{"eos_token"}]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1461
1462
1463
)


chenych's avatar
chenych committed
1464
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1465
1466
    name="openchat",
    format_user=StringFormatter(slots=["GPT4 Correct User: {{content}}", {"eos_token"}, "GPT4 Correct Assistant:"]),
chenych's avatar
chenych committed
1467
1468
1469
1470
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
)


chenych's avatar
chenych committed
1471
register_template(
chenych's avatar
chenych committed
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
    name="openchat-3.6",
    format_user=StringFormatter(
        slots=[
            (
                "<|start_header_id|>GPT4 Correct User<|end_header_id|>\n\n{{content}}<|eot_id|>"
                "<|start_header_id|>GPT4 Correct Assistant<|end_header_id|>\n\n"
            )
        ]
    ),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    stop_words=["<|eot_id|>"],
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1483
1484
1485
)


luopl's avatar
luopl committed
1486
# copied from chatml template
chenych's avatar
chenych committed
1487
register_template(
luopl's avatar
luopl committed
1488
1489
    name="opencoder",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1490
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1491
1492
1493
1494
1495
1496
1497
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_observation=StringFormatter(slots=["<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    default_system="You are OpenCoder, created by OpenCoder Team.",
    stop_words=["<|im_end|>"],
)


chenych's avatar
chenych committed
1498
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1499
1500
    name="orion",
    format_user=StringFormatter(slots=["Human: {{content}}\n\nAssistant: ", {"eos_token"}]),
chenych's avatar
chenych committed
1501
1502
1503
1504
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
)


chenych's avatar
chenych committed
1505
register_template(
luopl's avatar
luopl committed
1506
    name="paligemma",
chenych's avatar
chenych committed
1507
1508
1509
    format_user=StringFormatter(slots=["{{content}}\n"]),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    mm_plugin=get_mm_plugin(name="paligemma", image_token="<image>"),
chenych's avatar
chenych committed
1510
    template_class=Llama2Template,
chenych's avatar
chenych committed
1511
1512
1513
1514
1515
1516
)


# copied from gemma template
register_template(
    name="paligemma_chat",
luopl's avatar
luopl committed
1517
    format_user=StringFormatter(slots=["<start_of_turn>user\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]),
luopl's avatar
luopl committed
1518
    format_assistant=StringFormatter(slots=["{{content}}<end_of_turn>\n"]),
luopl's avatar
luopl committed
1519
1520
1521
1522
    format_observation=StringFormatter(
        slots=["<start_of_turn>tool\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]
    ),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
chenych's avatar
chenych committed
1523
    stop_words=["<end_of_turn>"],
chenych's avatar
chenych committed
1524
    replace_eos=True,
luopl's avatar
luopl committed
1525
    mm_plugin=get_mm_plugin(name="paligemma", image_token="<image>"),
chenych's avatar
chenych committed
1526
    template_class=Llama2Template,
luopl's avatar
luopl committed
1527
1528
1529
)


chenych's avatar
chenych committed
1530
register_template(
chenych's avatar
chenych committed
1531
1532
    name="phi",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|end|>\n<|assistant|>\n"]),
luopl's avatar
luopl committed
1533
    format_assistant=StringFormatter(slots=["{{content}}<|end|>\n"]),
chenych's avatar
chenych committed
1534
1535
    format_system=StringFormatter(slots=["<|system|>\n{{content}}<|end|>\n"]),
    stop_words=["<|end|>"],
chenych's avatar
chenych committed
1536
    replace_eos=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1537
1538
1539
)


chenych's avatar
chenych committed
1540
register_template(
luopl's avatar
luopl committed
1541
1542
    name="phi_small",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|end|>\n<|assistant|>\n"]),
luopl's avatar
luopl committed
1543
    format_assistant=StringFormatter(slots=["{{content}}<|end|>\n"]),
luopl's avatar
luopl committed
1544
1545
1546
    format_system=StringFormatter(slots=["<|system|>\n{{content}}<|end|>\n"]),
    format_prefix=EmptyFormatter(slots=[{"<|endoftext|>"}]),
    stop_words=["<|end|>"],
chenych's avatar
chenych committed
1547
    replace_eos=True,
luopl's avatar
luopl committed
1548
1549
1550
)


chenych's avatar
chenych committed
1551
register_template(
luopl's avatar
luopl committed
1552
1553
1554
1555
1556
1557
1558
    name="phi4",
    format_user=StringFormatter(
        slots=["<|im_start|>user<|im_sep|>{{content}}<|im_end|><|im_start|>assistant<|im_sep|>"]
    ),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>"]),
    format_system=StringFormatter(slots=["<|im_start|>system<|im_sep|>{{content}}<|im_end|>"]),
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1559
    replace_eos=True,
luopl's avatar
luopl committed
1560
1561
1562
)


chenych's avatar
chenych committed
1563
1564
# copied from ministral template
register_template(
luopl's avatar
luopl committed
1565
    name="pixtral",
luopl's avatar
luopl committed
1566
1567
    format_user=StringFormatter(slots=["[INST]{{content}}[/INST]"]),
    format_system=StringFormatter(slots=["{{content}}\n\n"]),
chenych's avatar
chenych committed
1568
1569
1570
    format_function=FunctionFormatter(slots=["[TOOL_CALLS]{{content}}", {"eos_token"}], tool_format="mistral"),
    format_observation=StringFormatter(slots=["""[TOOL_RESULTS]{"content": {{content}}}[/TOOL_RESULTS]"""]),
    format_tools=ToolFormatter(tool_format="mistral"),
luopl's avatar
luopl committed
1571
1572
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    mm_plugin=get_mm_plugin(name="pixtral", image_token="[IMG]"),
chenych's avatar
chenych committed
1573
    template_class=Llama2Template,
luopl's avatar
luopl committed
1574
1575
1576
)


luopl's avatar
luopl committed
1577
# copied from chatml template
chenych's avatar
chenych committed
1578
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1579
1580
    name="qwen",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1581
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1582
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1583
1584
1585
1586
1587
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
chenych's avatar
chenych committed
1588
    default_system="You are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1589
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1590
    replace_eos=True,
luopl's avatar
luopl committed
1591
1592
1593
)


chenych's avatar
chenych committed
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
# copied from qwen template
register_template(
    name="qwen3",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1606
1607
    replace_eos=True,
    template_class=ReasoningTemplate,
chenych's avatar
chenych committed
1608
1609
1610
)


luopl's avatar
luopl committed
1611
# copied from chatml template
chenych's avatar
chenych committed
1612
1613
1614
1615
1616
1617
1618
register_template(
    name="qwen2_audio",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    default_system="You are a helpful assistant.",
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1619
    replace_eos=True,
chenych's avatar
chenych committed
1620
1621
1622
1623
    mm_plugin=get_mm_plugin(name="qwen2_audio", audio_token="<|AUDIO|>"),
)


chenych's avatar
chenych committed
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
# copied from qwen template
register_template(
    name="qwen2_omni",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
    default_system="You are a helpful assistant.",
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1637
    replace_eos=True,
chenych's avatar
chenych committed
1638
1639
1640
1641
1642
    mm_plugin=get_mm_plugin(
        name="qwen2_omni", audio_token="<|AUDIO|>", image_token="<|IMAGE|>", video_token="<|VIDEO|>"
    ),
)

chenych's avatar
chenych committed
1643
1644
# copied from qwen template
register_template(
luopl's avatar
luopl committed
1645
1646
    name="qwen2_vl",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1647
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1648
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
luopl's avatar
luopl committed
1649
1650
1651
1652
1653
    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
    format_observation=StringFormatter(
        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
    ),
    format_tools=ToolFormatter(tool_format="qwen"),
luopl's avatar
luopl committed
1654
1655
    default_system="You are a helpful assistant.",
    stop_words=["<|im_end|>"],
chenych's avatar
chenych committed
1656
    replace_eos=True,
luopl's avatar
luopl committed
1657
    mm_plugin=get_mm_plugin(name="qwen2_vl", image_token="<|image_pad|>", video_token="<|video_pad|>"),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1658
1659
1660
)


chenych's avatar
chenych committed
1661
register_template(
chenych's avatar
chenych committed
1662
1663
    name="sailor",
    format_user=StringFormatter(slots=["<|im_start|>question\n{{content}}<|im_end|>\n<|im_start|>answer\n"]),
luopl's avatar
luopl committed
1664
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
chenych's avatar
chenych committed
1665
1666
1667
1668
1669
1670
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    default_system=(
        "You are an AI assistant named Sailor created by Sea AI Lab. "
        "Your answer should be friendly, unbiased, faithful, informative and detailed."
    ),
    stop_words=["<|im_end|>"],
luopl's avatar
luopl committed
1671
1672
1673
)


chenych's avatar
chenych committed
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
register_template(
    name="seed_coder",
    format_user=StringFormatter(
        slots=[{"bos_token"}, "user\n{{content}}", {"eos_token"}, {"bos_token"}, "assistant\n"]
    ),
    format_system=StringFormatter(slots=[{"bos_token"}, "system\n{{content}}", {"eos_token"}]),
    default_system=(
        "You are an AI programming assistant, utilizing the Seed-Coder model, developed by ByteDance Seed, "
        "and you only answer questions related to computer science. For politically sensitive questions, "
        "security and privacy issues, and other non-computer science questions, you will refuse to answer.\n\n"
    ),
)


luopl's avatar
luopl committed
1688
# copied from llama3 template
chenych's avatar
chenych committed
1689
register_template(
luopl's avatar
luopl committed
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
    name="skywork_o1",
    format_user=StringFormatter(
        slots=[
            (
                "<|start_header_id|>user<|end_header_id|>\n\n{{content}}<|eot_id|>"
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
    format_assistant=StringFormatter(slots=["{{content}}<|eot_id|>"]),
    format_system=StringFormatter(slots=["<|start_header_id|>system<|end_header_id|>\n\n{{content}}<|eot_id|>"]),
    format_function=FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3"),
    format_observation=StringFormatter(
        slots=[
            (
                "<|start_header_id|>ipython<|end_header_id|>\n\n{{content}}<|eot_id|>"
                "<|start_header_id|>assistant<|end_header_id|>\n\n"
            )
        ]
    ),
    format_tools=ToolFormatter(tool_format="llama3"),
    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
    default_system=(
        "You are Skywork-o1, a thinking model developed by Skywork AI, specializing in solving complex problems "
        "involving mathematics, coding, and logical reasoning through deep thought. When faced with a user's request, "
        "you first engage in a lengthy and in-depth thinking process to explore possible solutions to the problem. "
        "After completing your thoughts, you then provide a detailed explanation of the solution process "
        "in your response."
    ),
    stop_words=["<|eot_id|>", "<|eom_id|>"],
chenych's avatar
chenych committed
1720
1721
1722
)


chenych's avatar
chenych committed
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
register_template(
    name="smollm",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    stop_words=["<|im_end|>"],
)


register_template(
    name="smollm2",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
    stop_words=["<|im_end|>"],
    default_system="You are a helpful AI assistant named SmolLM, trained by Hugging Face.",
)


chenych's avatar
chenych committed
1742
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1743
1744
1745
1746
1747
1748
1749
    name="solar",
    format_user=StringFormatter(slots=["### User:\n{{content}}\n\n### Assistant:\n"]),
    format_system=StringFormatter(slots=["### System:\n{{content}}\n\n"]),
    efficient_eos=True,
)


chenych's avatar
chenych committed
1750
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1751
1752
    name="starchat",
    format_user=StringFormatter(slots=["<|user|>\n{{content}}<|end|>\n<|assistant|>"]),
luopl's avatar
luopl committed
1753
    format_assistant=StringFormatter(slots=["{{content}}<|end|>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1754
1755
    format_system=StringFormatter(slots=["<|system|>\n{{content}}<|end|>\n"]),
    stop_words=["<|end|>"],
chenych's avatar
chenych committed
1756
1757
1758
)


chenych's avatar
chenych committed
1759
register_template(
chenych's avatar
chenych committed
1760
1761
1762
    name="telechat",
    format_user=StringFormatter(slots=["<_user>{{content}}<_bot>"]),
    format_system=StringFormatter(slots=["<_system>{{content}}<_end>"]),
luopl's avatar
luopl committed
1763
1764
1765
)


chenych's avatar
chenych committed
1766
register_template(
luopl's avatar
luopl committed
1767
1768
1769
1770
1771
1772
    name="telechat2",
    format_user=StringFormatter(slots=["<_user>{{content}}<_bot>"]),
    format_system=StringFormatter(slots=["<_system>{{content}}"]),
    default_system=(
        "你是中国电信星辰语义大模型,英文名是TeleChat,你是由中电信人工智能科技有限公司和中国电信人工智能研究院(TeleAI)研发的人工智能助手。"
    ),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1773
1774
1775
)


chenych's avatar
chenych committed
1776
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1777
1778
1779
1780
1781
1782
    name="vicuna",
    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
    default_system=(
        "A chat between a curious user and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the user's questions."
    ),
luopl's avatar
luopl committed
1783
    replace_jinja_template=True,
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1784
1785
1786
)


chenych's avatar
chenych committed
1787
register_template(
luopl's avatar
luopl committed
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
    name="video_llava",
    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
    default_system=(
        "A chat between a curious user and an artificial intelligence assistant. "
        "The assistant gives helpful, detailed, and polite answers to the user's questions."
    ),
    mm_plugin=get_mm_plugin(name="video_llava", image_token="<image>", video_token="<video>"),
)


chenych's avatar
chenych committed
1798
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
    name="xuanyuan",
    format_user=StringFormatter(slots=["Human: {{content}} Assistant:"]),
    default_system=(
        "以下是用户和人工智能助手之间的对话。用户以Human开头,人工智能助手以Assistant开头,"
        "会对人类提出的问题给出有帮助、高质量、详细和礼貌的回答,并且总是拒绝参与与不道德、"
        "不安全、有争议、政治敏感等相关的话题、问题和指示。\n"
    ),
)


chenych's avatar
chenych committed
1809
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1810
1811
1812
1813
1814
    name="xverse",
    format_user=StringFormatter(slots=["Human: {{content}}\n\nAssistant: "]),
)


chenych's avatar
chenych committed
1815
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1816
1817
    name="yayi",
    format_user=StringFormatter(slots=[{"token": "<|Human|>"}, ":\n{{content}}\n\n", {"token": "<|YaYi|>"}, ":"]),
luopl's avatar
luopl committed
1818
    format_assistant=StringFormatter(slots=["{{content}}\n\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
    format_system=StringFormatter(slots=[{"token": "<|System|>"}, ":\n{{content}}\n\n"]),
    default_system=(
        "You are a helpful, respectful and honest assistant named YaYi "
        "developed by Beijing Wenge Technology Co.,Ltd. "
        "Always answer as helpfully as possible, while being safe.  "
        "Your answers should not include any harmful, unethical, "
        "racist, sexist, toxic, dangerous, or illegal content. "
        "Please ensure that your responses are socially unbiased and positive in nature.\n\n"
        "If a question does not make any sense, or is not factually coherent, "
        "explain why instead of answering something not correct. "
        "If you don't know the answer to a question, please don't share false information."
    ),
    stop_words=["<|End|>"],
)


luopl's avatar
luopl committed
1835
# copied from chatml template
chenych's avatar
chenych committed
1836
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1837
1838
    name="yi",
    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
luopl's avatar
luopl committed
1839
    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
chenych's avatar
chenych committed
1840
    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1841
1842
1843
1844
    stop_words=["<|im_end|>"],
)


chenych's avatar
chenych committed
1845
register_template(
chenych's avatar
chenych committed
1846
1847
    name="yi_vl",
    format_user=StringFormatter(slots=["### Human: {{content}}\n### Assistant:"]),
luopl's avatar
luopl committed
1848
    format_assistant=StringFormatter(slots=["{{content}}\n"]),
chenych's avatar
chenych committed
1849
1850
1851
1852
1853
1854
1855
1856
1857
    default_system=(
        "This is a chat between an inquisitive human and an AI assistant. "
        "Assume the role of the AI assistant. Read all the images carefully, "
        "and respond to the human's questions with informative, helpful, detailed and polite answers. "
        "这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。"
        "仔细阅读所有的图像,并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。\n\n"
    ),
    stop_words=["###"],
    efficient_eos=True,
luopl's avatar
luopl committed
1858
    mm_plugin=get_mm_plugin(name="llava", image_token="<image>"),
chenych's avatar
chenych committed
1859
1860
1861
)


chenych's avatar
chenych committed
1862
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1863
1864
    name="yuan",
    format_user=StringFormatter(slots=["{{content}}", {"token": "<sep>"}]),
luopl's avatar
luopl committed
1865
    format_assistant=StringFormatter(slots=["{{content}}<eod>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1866
1867
1868
1869
    stop_words=["<eod>"],
)


chenych's avatar
chenych committed
1870
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1871
    name="zephyr",
chenych's avatar
chenych committed
1872
    format_user=StringFormatter(slots=["<|user|>\n{{content}}", {"eos_token"}, "<|assistant|>\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1873
    format_system=StringFormatter(slots=["<|system|>\n{{content}}", {"eos_token"}]),
chenych's avatar
chenych committed
1874
    default_system="You are Zephyr, a helpful assistant.",
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1875
1876
1877
)


chenych's avatar
chenych committed
1878
register_template(
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1879
1880
    name="ziya",
    format_user=StringFormatter(slots=["<human>:{{content}}\n<bot>:"]),
luopl's avatar
luopl committed
1881
    format_assistant=StringFormatter(slots=["{{content}}\n"]),
Rayyyyy's avatar
V0.6.3  
Rayyyyy committed
1882
)