added checks for if gold is outside choice options

lm_eval/api/task.py

@@ -1014,18 +1014,36 @@ class ConfigurableTask(Task):
                 gold = self.doc_to_text(doc)
             else:
                 gold = self.doc_to_target(doc)
-                if type(gold) is str:
-                    gold = choices.index(gold)
+
+            gold_index_error = False
+            if type(gold) is list:
+                gold = [i if i < len(choices) else -1000 for i in gold]
+                if -1000 in gold:
+                    gold_index_error = True
+            else:
+                if type(gold) is int:
+                    gold = gold if i < len(choice) else -1000
+                elif type(gold) is str:
+                    gold = choices.index(gold) if gold in choices else -1000                    
+                
+                if gold == -1000:
+                    gold_index_error = True
+
+            if gold_index_error:
+                eval_logger.warning(
+                    f"Label index was not in within range of availabel choices,"
+                    f"Sample:\n\n{doc}\n\n"
+                )
 
             if self.multiple_target:
                 acc = 1.0 if pred in gold else 0.0
                 acc_norm = 1.0 if pred_norm in gold else 0.0
-                exact_match = int(any([is_greedy[i] for i in gold]))
+                exact_match = int(any([is_greedy[i] if i != -1000 else 0 for i in gold]))
             else:
                 acc = 1.0 if pred == gold else 0.0
                 acc_norm = 1.0 if pred_norm == gold else 0.0
                 # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly
-                exact_match = int(is_greedy[gold])
+                exact_match = int(is_greedy[gold]) if i != -1000 else 0 
 
             result_dict = {
                 **({"acc": acc} if "acc" in use_metric else {}),