Moved apply chat template to LM

lm_eval/api/model.py

@@ -114,6 +114,19 @@ class LM(abc.ABC):
         """
         pass
 
+    @abc.abstractmethod
+    def apply_chat_template(self, chat_history: list[dict[str, str]]) -> str:
+        """
+        Defines how to transform few-shot examples provided as chat history into a format that can be used as input to the LM.
+
+        :param chat_history: list[dict[str, str]]
+            A list of dictionaries with keys 'role' and 'content'.
+            Values are strings representing the role name and the content of the message, respectively.
+        :return: str
+            A string representing the chat history in a format that can be used as input to the LM.
+        """
+        pass
+
     @classmethod
     def create_from_arg_string(
         cls: Type[T], arg_string: str, additional_config: Optional[dict] = None

lm_eval/api/task.py

@@ -376,7 +376,7 @@ class Task(abc.ABC):
         system_instruction=None,
         apply_chat_template=False,
         fewshot_as_multiturn=False,
-        tokenizer=None,
+        lm=None,
     ) -> None:
         """Build a set of Instances for a task, and store them in task.instances"""
 
@@ -428,7 +428,7 @@ class Task(abc.ABC):
                 system_instruction,
                 apply_chat_template,
                 fewshot_as_multiturn,
-                tokenizer,
+                lm,
             )
 
             # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute
@@ -965,24 +965,6 @@ class ConfigurableTask(Task):
                 )
             return super().fewshot_docs()
 
-    def convert_chat_history_to_string(self, chat_history: list, tokenizer=None) -> str:
-        """Returns chat history tokenized or concatenated as a string.
-
-        :param chat_history: list
-            The chat history to convert to a string.
-        :param tokenizer:
-            Optional tokenizer to use for applying the chat template, if None, the sampler's fewshot_delimiter is used.
-        """
-        if tokenizer:
-            return tokenizer.apply_chat_template(
-                chat_history, tokenize=False, add_generation_prompt=True
-            )
-        else:
-            return self.sampler.fewshot_delimiter + "".join(
-                f"{s['role']}: {s['content']}" + self.sampler.fewshot_delimiter
-                for s in chat_history
-            )
-
     @utils.positional_deprecated
     def fewshot_context(
         self,
@@ -991,7 +973,7 @@ class ConfigurableTask(Task):
         system_instruction: Optional[str] = None,
         apply_chat_template: bool = False,
         fewshot_as_multiturn: bool = False,
-        tokenizer=None,
+        lm=None,
     ) -> str:
         """Returns a fewshot context string that is made up of a prepended description
         (if provided), the `num_fewshot` number of examples, and an appended prompt example.
@@ -1006,8 +988,8 @@ class ConfigurableTask(Task):
             Whether to apply the chat template to the fewshot context.
         :param fewshot_as_multiturn: bool
             Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
-        :param tokenizer:
-            The tokenizer to use for applying the chat template.
+        :param lm:
+            Language model with definition of the tokenizer/function to use for applying the chat template.
         :returns: str
             The fewshot context.
         """
@@ -1063,9 +1045,7 @@ class ConfigurableTask(Task):
                     for ex in example:
                         chat = deepcopy(labeled_examples)
                         chat.append({"role": "user", "content": ex})
-                        labeled_examples_list.append(
-                            self.convert_chat_history_to_string(chat, tokenizer)
-                        )
+                        labeled_examples_list.append(lm.apply_chat_template(chat))
                     return labeled_examples_list
                 # if example is an integer, append the choice or convert to string
                 elif isinstance(example, int):
@@ -1078,7 +1058,7 @@ class ConfigurableTask(Task):
                         labeled_examples.append(
                             {"role": "user", "content": str(example)}
                         )
-            return self.convert_chat_history_to_string(labeled_examples, tokenizer)
+            return lm.apply_chat_template(labeled_examples)
         else:
             if self.multiple_input:
                 return labeled_examples

lm_eval/evaluator.py

@@ -386,7 +386,7 @@ def evaluate(
             system_instruction=system_instruction,
             apply_chat_template=apply_chat_template,
             fewshot_as_multiturn=fewshot_as_multiturn,
-            tokenizer=lm.tokenizer if hasattr(lm, "tokenizer") else None,
+            lm=lm,
         )
         eval_logger.debug(
             f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"

lm_eval/models/huggingface.py

@@ -1286,6 +1286,14 @@ class HFLM(TemplateLM):
 
         return res
 
+    def apply_chat_template(self, chat_history: list[dict[str, str]]) -> str:
+        """
+        Method to apply a chat template to a list of chat history between user and model.
+        """
+        return self.tokenizer.apply_chat_template(
+            chat_history, tokenize=False, add_generation_prompt=True
+        )
+
     def get_model_info(self) -> dict:
         """
         Method to get Hugging Face model information for experiment reproducibility.