Skip to content

GitLab

Unverified Commit 6463070b authored by wang jiahao's avatar wang jiahao Committed by GitHub
Browse files

Merge pull request #1064 from kvcache-ai/fix-temperature 

fix temperature=0, flashinfer sample error
parents aac0c91d ec03bcbd
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 0 deletions
ktransformers/server/backend/interfaces/balance_serve.py
View file @ 6463070b
...@@ -367,6 +367,8 @@ class BalanceServeInterface(BackendInterfaceBase): ...@@ -367,6 +367,8 @@ class BalanceServeInterface(BackendInterfaceBase):
#@TODO add server #@TODO add server
stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")] stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")]
query_add.stop_criteria = stop_criteria query_add.stop_criteria = stop_criteria
if temperature == 0:
temperature = 0.0001
query_add.sample_options.temperature = temperature query_add.sample_options.temperature = temperature
if top_p == 0: if top_p == 0:
top_p = 0.0001 top_p = 0.0001
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment