remove tokenizer assert

5f1d18d4 · haileyschoelkopf · 2a92cde0 · 5f1d18d4
Commit 5f1d18d4 authored Jun 27, 2023 by haileyschoelkopf
Hide whitespace changes
Inline Side-by-side

Showing with 0 additions and 10 deletions

lm_eval/models/gpt2.py lm_eval/models/gpt2.py +0 -10

No files found.
--- a/lm_eval/models/gpt2.py
+++ b/lm_eval/models/gpt2.py
@@ -102,16 +102,6 @@ class HFLM(BaseLM):

        self.vocab_size = self.tokenizer.vocab_size

-        if isinstance(
-            self.tokenizer, (transformers.GPT2Tokenizer, transformers.GPT2TokenizerFast)
-        ):
-            assert self.tokenizer.encode("hello\n\nhello") == [
-                31373,
-                198,
-                198,
-                31373,
-            ], self.tokenizer.encode("hello\n\nhello")
-
        # Validate batch_size
        assert isinstance(batch_size, (int, str))