Merge pull request #343 from EleutherAI/fix-tokenizer-init

Manually concat tokenizer revision with subfolder

lm_eval/models/gpt2.py

@@ -34,17 +34,17 @@ class HFLM(BaseLM):
             )
 
         # TODO: update this to be less of a hack once subfolder is fixed in HF
+        revision = revision + ("/" + subfolder if subfolder is not None else "")
+
         self.gpt2 = transformers.AutoModelForCausalLM.from_pretrained(
             pretrained,
-            revision=revision + ("/" + subfolder if subfolder is not None else ""),
+            revision=revision,
         ).to(self.device)
         self.gpt2.eval()
 
-        # pretrained tokenizer for neo is broken for now so just hard-coding this to gpt2
         self.tokenizer = transformers.AutoTokenizer.from_pretrained(
             pretrained if tokenizer is None else tokenizer,
             revision=revision,
-            subfolder=subfolder,
         )
 
         assert isinstance(