Merge pull request #584 from gakada/perf

Fix non-callable attributes in CachingLM

lm_eval/base.py

@@ -868,6 +868,10 @@ class CachingLM:
         lm.set_cache_hook(self.get_cache_hook())
 
     def __getattr__(self, attr):
+        lm_attr = getattr(self.lm, attr)
+        if not callable(lm_attr):
+            return lm_attr
+
         def fn(requests):
             res = []
             remaining_reqs = []

lm_eval/evaluator.py

@@ -109,7 +109,7 @@ def simple_evaluate(
         "model_args": model_args,
         "num_fewshot": num_fewshot,
         "batch_size": batch_size,
-        "batch_sizes": list(lm.batch_sizes.values()),
+        "batch_sizes": list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else [],
         "device": device,
         "no_cache": no_cache,
         "limit": limit,