doc_to_target should also be able to output str instead of solely int

lm_eval/api/samplers.py

@@ -42,7 +42,10 @@ class Sampler:
                     + self.target_delimiter
                     + (
                         self.doc_to_target(doc)
-                        if self.config.doc_to_choice is None
+                        if (
+                            self.config.doc_to_choice is None
+                            or type(self.doc_to_target(doc)) is str
+                        )
                         else self.doc_to_choice(doc)[self.doc_to_target(doc)]
                     )
                     for doc in selected_docs

lm_eval/api/task.py

@@ -946,6 +946,8 @@ class ConfigurableTask(Task):
                 gold = self.doc_to_text(doc)
             else:
                 gold = self.doc_to_target(doc)
+                if type(gold) is str:
+                    gold = choices.index(gold)
 
             if self.multiple_target:
                 acc = 1.0 if pred in gold else 0.0