feat / fix: Properly make use of `subfolder` from HF models (#3072)

* add subfolder

* lint

* change it to empty string

* fix typehints

---------
Co-authored-by: Baber <baber@hey.com>

lm_eval/models/huggingface.py

@@ -61,7 +61,7 @@ class HFLM(TemplateLM):
         backend: Literal["default", "causal", "seq2seq"] = "default",
         # override whether the model should be treated as decoder-only (causal) or encoder-decoder (seq2seq)
         revision: Optional[str] = "main",
-        subfolder: Optional[str] = None,
+        subfolder: str = "",
         tokenizer: Optional[
             Union[
                 str,
@@ -162,14 +162,13 @@ class HFLM(TemplateLM):
                 )
 
             revision = str(revision)  # cast to string if not already one
-            # TODO: update this to be less of a hack once subfolder is fixed in HF
-            revision = revision + ("/" + subfolder if subfolder is not None else "")
 
             self._get_config(
                 pretrained,
                 revision=revision,
                 trust_remote_code=trust_remote_code,
                 gguf_file=gguf_file,
+                subfolder=subfolder,
             )
 
             # determine which of 'causal' and 'seq2seq' backends to use for HF models
@@ -182,6 +181,7 @@ class HFLM(TemplateLM):
             pretrained,
             tokenizer,
             revision=revision,
+            subfolder=subfolder,
             trust_remote_code=trust_remote_code,
             use_fast_tokenizer=use_fast_tokenizer,
             gguf_file=gguf_file,
@@ -206,6 +206,7 @@ class HFLM(TemplateLM):
                 gptqmodel=gptqmodel,
                 gguf_file=gguf_file,
                 quantization_config=getattr(self.config, "quantization_config", None),
+                subfolder=subfolder,
                 **kwargs,
             )
 
@@ -522,6 +523,7 @@ class HFLM(TemplateLM):
         revision: str = "main",
         trust_remote_code: bool = False,
         gguf_file: Optional[str] = None,
+        subfolder: str = "",
     ) -> None:
         """Return the model config for HuggingFace models"""
         self._config = transformers.AutoConfig.from_pretrained(
@@ -529,6 +531,7 @@ class HFLM(TemplateLM):
             revision=revision,
             trust_remote_code=trust_remote_code,
             gguf_file=gguf_file,
+            subfolder=subfolder,
         )
 
     def _create_model(
@@ -552,6 +555,7 @@ class HFLM(TemplateLM):
         gptqmodel: Optional[bool] = False,
         gguf_file: Optional[str] = None,
         quantization_config: Optional[Dict[str, Any]] = None,
+        subfolder: str = "",
         **kwargs,
     ) -> None:
         """
@@ -598,6 +602,7 @@ class HFLM(TemplateLM):
                 trust_remote_code=trust_remote_code,
                 gguf_file=gguf_file,
                 quantization_config=quantization_config,
+                subfolder=subfolder,
                 **model_kwargs,
             )
         else:
@@ -697,6 +702,7 @@ class HFLM(TemplateLM):
         use_fast_tokenizer: Optional[bool] = True,
         gguf_file: Optional[str] = None,
         add_bos_token: Optional[bool] = False,
+        subfolder: Optional[str] = "",
     ) -> None:
         """
         Helper method during initialization.
@@ -718,6 +724,9 @@ class HFLM(TemplateLM):
         if add_bos_token:
             kwargs["add_bos_token"] = True
 
+        if subfolder:
+            kwargs["subfolder"] = subfolder
+
         if tokenizer:
             if isinstance(tokenizer, str):
                 self.tokenizer = transformers.AutoTokenizer.from_pretrained(