llama test

lm_eval/models/huggingface.py

@@ -673,18 +673,21 @@ class HFLM(LM):
             chat = [
               {"role": "system", "content": "You are a helpful, respectful and honest assistant."}, 
               {"role": "user", "content": context},
+              {"role": "assistant", "content": continuation}, 
             ]
-            context = self.tokenizer.apply_chat_template(
+            single_tokenized_conversation = self.tokenizer.apply_chat_template(
                 chat, 
                 tokenize=False,
                 add_generation_prompt=True,
             )
+            rfind_continuation = single_tokenized_conversation.rfind(continuation)
+            context = single_tokenized_conversation[:rfind_continuation]
+            continuation = single_tokenized_conversation[rfind_continuation:]
             req.args = (context, continuation) 
             new_reqs.append(req)
     
         return new_reqs
     
-    
     def _model_call(self, inps, attn_mask=None, labels=None):
         """
         :param inps: torch.Tensor