nit

1f27bd87 · Baber · bd34e852 · 1f27bd87
Commit 1f27bd87 authored Dec 19, 2024 by Baber
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

lm_eval/tasks/ruler/utils.py lm_eval/tasks/ruler/utils.py +3 -1

No files found.
--- a/lm_eval/tasks/ruler/utils.py
+++ b/lm_eval/tasks/ruler/utils.py
@@ -15,7 +15,9 @@ from lm_eval.tasks.ruler.prepare import generate_samples
 @cache
 def get_tokenizer():
-    return AutoTokenizer.from_pretrained(os.environ.get("TOKENIZER"))
+    return AutoTokenizer.from_pretrained(
+        os.environ.get("TOKENIZER"), trust_remote_code=True
+    )
 # TOKENIZER = AutoTokenizer.from_pretrained(os.environ.get("TOKENIZER"))