Merge pull request #490 from jquesnelle/auto-batch-size-fix

fix adaptive batch crash when there are no new requests

lm_eval/base.py

@@ -254,25 +254,28 @@ class BaseLM(LM):
 
         # automatic (variable) batch size detection for vectorization
         # pull longest context sample from request
-        _, context_enc, continuation_enc = re_ord.get_reordered()[0]
-        max_context = len((context_enc + continuation_enc)[-(self.max_length + 1) :][:-1])
-        if (self.batch_size == 'auto'):
-            
-            if override_bs is None:
-                print('Passed argument batch_size = auto. Detecting largest batch size')
-                @find_executable_batch_size(starting_batch_size=512) # if OOM, then halves batch_size and tries again
-                def forward_batch(batch_size):
-                    test_batch = torch.ones((batch_size, max_context), device=self.device).long()
-                    for _ in range(5): 
-                        out = F.log_softmax(self._model_call(test_batch), dim = -1).cpu()
-                    return batch_size
-                
-                batch_size = forward_batch() 
-                print(f"Determined largest batch size: {batch_size}")
-                adaptive_batch_size = batch_size
+        if len(re_ord.get_reordered()) > 0:
+            _, context_enc, continuation_enc = re_ord.get_reordered()[0]
+            max_context = len((context_enc + continuation_enc)[-(self.max_length + 1) :][:-1])
+            if (self.batch_size == 'auto'):
                 
-            else:
-                adaptive_batch_size = override_bs
+                if override_bs is None:
+                    print('Passed argument batch_size = auto. Detecting largest batch size')
+                    @find_executable_batch_size(starting_batch_size=512) # if OOM, then halves batch_size and tries again
+                    def forward_batch(batch_size):
+                        test_batch = torch.ones((batch_size, max_context), device=self.device).long()
+                        for _ in range(5):
+                            out = F.log_softmax(self._model_call(test_batch), dim = -1).cpu()
+                        return batch_size
+
+                    batch_size = forward_batch()
+                    print(f"Determined largest batch size: {batch_size}")
+                    adaptive_batch_size = batch_size
+
+                else:
+                    adaptive_batch_size = override_bs
+        else:
+            adaptive_batch_size = 0 if override_bs is None else override_bs
 
         for chunk in utils.chunks(
             tqdm(re_ord.get_reordered(), disable=disable_tqdm), self.batch_size if self.batch_size != "auto" else adaptive_batch_size