Commit d8f03ffe authored by zhuwenwen's avatar zhuwenwen
Browse files

add llama/llama_tokenizer.py

parent a0382aa6
import transformers
import torch
from transformers import LlamaForCausalLM, LlamaTokenizer
tokenizer = LlamaTokenizer.from_pretrained('/data/models/llama-7b-hf/')
with open('./out', 'r') as file:
output = file.read()
output = torch.tensor([int(id_) for id_ in output.split()])
output = tokenizer.batch_decode(output.unsqueeze(0), skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
print(output)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment