set fp32 if device=mps

lm_eval/api/model.py

@@ -114,6 +114,8 @@ class LM(abc.ABC):
         additional_config = {} if additional_config is None else additional_config
         args = utils.simple_parse_args_string(arg_string)
         args2 = {k: v for k, v in additional_config.items() if v is not None}
+        if "device" in args and args["device"] == "mps":
+            args["dtype"] = "float32"
         return cls(**args, **args2)
 
     @property

lm_eval/models/huggingface.py

@@ -109,7 +109,7 @@ class HFLM(LM):
                 eval_logger.info(f"Using device '{device}'")
                 if device == "mps":
                     eval_logger.info(
-                        "MPS is still in beta; add ,dtype=float32 to model_args ."
+                        "MPS is still in beta and only supports float32; setting dtype to float32."
                     )
             else:
                 eval_logger.info("Device not specified")