Commit 306af132 authored by thomwolf's avatar thomwolf
Browse files

update readme to mention add_special_tokens more clearly in example

parent 50e6daf8
......@@ -93,7 +93,7 @@ for model_class, tokenizer_class, pretrained_weights in MODELS:
model = model_class.from_pretrained(pretrained_weights)
# Encode text
input_ids = torch.tensor([tokenizer.encode("Here is some text to encode")])
input_ids = torch.tensor([tokenizer.encode("Here is some text to encode", add_special_tokens=True)]) # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.
with torch.no_grad():
last_hidden_states = model(input_ids)[0] # Models outputs are now tuples
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment