first stab at wrap_chat_template, arc conversation test

lm_eval/evaluator.py

@@ -326,7 +326,6 @@ def evaluate(
 
         # put responses from model into a list of length K for each request.
         for x, req in zip(resps, cloned_reqs):
-            print(x)
             req.resps.append(x)
 
         if lm.world_size > 1:

lm_eval/models/huggingface.py

@@ -667,13 +667,32 @@ class HFLM(LM):
         """
         Utility for adding chat templates via the apply_chat_template() method
         """
+        import re
+
         new_reqs = []
         for req in requests:
             context, continuation = req.args[0].strip(), req.args[1].strip()
-            chat = [
-                #{"role": "system", "content": "You are a helpful, respectful and honest assistant."},
-                {"role": "user", "content": context},
-            ]
+
+            # Regex pattern splits on substrings "; " and ", "
+            elements = re.split('Answer: |Question: ', context.replace('\n', ' '))
+            
+            new_elements = []
+            for element in elements[1:]:
+                new_elements.append(element.strip())
+            
+            new_elements
+            chat = []
+            
+            for i in range(len(new_elements)):
+                if i % 2 == 0:
+                    chat.append({"role": "user", "content": f"Question: {new_elements[i]} Answer:"})
+                else:
+                    chat.append({"role": "assistant", "content": f"{new_elements[i]}"})
+
+            #chat = [
+            #    #{"role": "system", "content": "You are a helpful, respectful and honest assistant."},
+            #    {"role": "user", "content": context},
+            #]
             context = self.tokenizer.apply_chat_template(
                 chat, 
                 tokenize=False,
@@ -681,7 +700,7 @@ class HFLM(LM):
             )
             req.args = (context, continuation) 
             new_reqs.append(req)
-
+    
         return new_reqs