Merge pull request #802 from EleutherAI/fix-metrics

Merge Fix metrics branch

ignore.txt

@@ -4,4 +4,4 @@ nin
 maka
 mor
 te
-ond
\ No newline at end of file
+ond

lm_eval/api/task.py

@@ -659,14 +659,14 @@ class ConfigurableTask(Task):
             self.multiple_target = len(test_target)
         else:
             if (type(test_target) is int) and (test_choice is not None):
-                test_target = [self.doc_to_choice(test_target)[test_target]]
+                test_target = test_choice[test_target]
             else:
-                test_target = [test_target]
+                test_target = str(test_target)
 
         if test_choice is not None:
             check_choices = test_choice
         else:
-            check_choices = test_target
+            check_choices = [test_target]
 
         for choice in check_choices:
             choice_has_whitespace = True if " " in choice else False

lm_eval/evaluator.py

@@ -219,7 +219,6 @@ def evaluate(
     padding_requests = collections.defaultdict(int)
 
     # Stores group related keys and values for group-aggregation
-    aggregate = collections.defaultdict(dict)
     task_groups = collections.defaultdict(dict)
 
     # get lists of each type of request
@@ -228,6 +227,7 @@ def evaluate(
         if type(task) == tuple:
             group, task = task
             task_groups[task_name] = group
+            aggregate[task_name] = {}
 
         versions[task_name] = task.VERSION
         configs[task_name] = dict(task.dump_config())
@@ -407,12 +407,12 @@ def evaluate(
             #        | word_perplexity
             #        | byte_perplexity
             #        | bits_per_byte
-            if bool(task_groups):
+            if task_name in task_groups:
                 group_name = task_groups[task_name]
-                if metric not in aggregate[group_name]:
-                    aggregate[group_name][metric] = [task_score]
-                else:
+                if metric in list(aggregate[group_name].keys()):
                     aggregate[group_name][metric].append(task_score)
+                else:
+                    aggregate[group_name][metric] = [task_score]
 
             # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap
             # so we run them less iterations. still looking for a cleaner way to do this