Commit b68561a0 authored by shihm's avatar shihm
Browse files

add inference.py

parent f2b3a0bd
...@@ -10,7 +10,7 @@ model = AutoModelForCausalLM.from_pretrained( ...@@ -10,7 +10,7 @@ model = AutoModelForCausalLM.from_pretrained(
device_map="auto", device_map="auto",
dtype=torch.bfloat16 dtype=torch.bfloat16
) )
enizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
messages = [{"role": "user", "content": "I've been having headaches lately, especially worse in the afternoon. What should I do?"}] messages = [{"role": "user", "content": "I've been having headaches lately, especially worse in the afternoon. What should I do?"}]
text = tokenizer.apply_chat_template( text = tokenizer.apply_chat_template(
messages, messages,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment