load_tokenizer.py 156 Bytes
Newer Older
Lianmin Zheng's avatar
Lianmin Zheng committed
1
2
3
4
5
6
7
import transformers
import code

name = "meta-llama/Llama-2-7b-chat-hf"

t = transformers.AutoTokenizer.from_pretrained(name)
code.interact(local=locals())