working draft

lm_eval/models/llava.py

@@ -198,6 +198,8 @@ class Llava(LM):
             total=len(requests), disable=(self.rank != 0), desc="Model Responding"
         )
 
+        print(requests[0])
+        import sys; sys.exit()
         for contexts, doc_to_target, doc_to_visual, doc, task in [
             reg.args for reg in requests
         ]:
@@ -322,10 +324,10 @@ class Llava(LM):
         )
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
         for chunk in chunks:
-            contexts, all_gen_kwargs, doc_to_visual, doc, task = zip(*chunk)
-            task = task[0]
+            contexts, all_gen_kwargs, visuals = zip(*chunk)
+            # task = task[0]
 
-            visuals = [doc_to_visual[0](doc[0])]
+            # visuals = [doc_to_visual[0](doc[0])]
             visuals = self.flatten(visuals)
             # we assume all gen kwargs in the batch are the same
             # this is safe to assume because the `grouper` object ensures it.

lm_eval/tasks/mmmu/_template_yaml

@@ -18,7 +18,7 @@ generation_kwargs:
   repetition_penalty: 1.0
   image_aspect_ratio: original
 metric_list:
-  - metric: acc
+  - metric: exact_match
   # - metric: mmmu_acc
   #   aggregation: !function utils.mmmu_aggregate_results
   #   higher_is_better: true