Commit 44e4196b authored by kabbi159's avatar kabbi159
Browse files

resolve TypeError

parent e265bd7c
...@@ -25,7 +25,8 @@ class HFLM(BaseLM): ...@@ -25,7 +25,8 @@ class HFLM(BaseLM):
# pretrained tokenizer for neo is broken for now so just hard-coding this to gpt2 # pretrained tokenizer for neo is broken for now so just hard-coding this to gpt2
self.tokenizer = transformers.AutoTokenizer.from_pretrained( self.tokenizer = transformers.AutoTokenizer.from_pretrained(
pretrained if tokenizer is None else tokenizer, revision=revision, subfolder=subfolder) pretrained if tokenizer is None else tokenizer,
revision=revision + ("/" + subfolder if subfolder is not None else ""))
# assert isinstance(self.tokenizer, ( # assert isinstance(self.tokenizer, (
# transformers.GPT2Tokenizer, transformers.GPT2TokenizerFast, # transformers.GPT2Tokenizer, transformers.GPT2TokenizerFast,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment